mirror of
https://github.com/dunwu/linux-tutorial.git
synced 2024-04-15 19:55:24 +08:00
167 lines
63 KiB
HTML
167 lines
63 KiB
HTML
|
<!DOCTYPE html>
|
|||
|
|
<html lang="en-US">
|
|||
|
|
<head>
|
|||
|
|
<meta charset="utf-8">
|
|||
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|||
|
|
<title>Elastic 技术栈之 Logstash 基础 | LINUX-TUTORIAL</title>
|
|||
|
|
<meta name="generator" content="VuePress 1.8.2">
|
|||
|
|
<link rel="icon" href="/linux-tutorial/favicon.ico">
|
|||
|
|
<meta name="description" content="数据库教程">
|
|||
|
|
|
|||
|
|
<link rel="preload" href="/linux-tutorial/assets/css/0.styles.45d9d031.css" as="style"><link rel="preload" href="/linux-tutorial/assets/js/app.79a38eea.js" as="script"><link rel="preload" href="/linux-tutorial/assets/js/4.fb6e0f89.js" as="script"><link rel="preload" href="/linux-tutorial/assets/js/53.76541550.js" as="script"><link rel="preload" href="/linux-tutorial/assets/js/5.cb43ecfb.js" as="script"><link rel="prefetch" href="/linux-tutorial/assets/js/10.7933187b.js"><link rel="prefetch" href="/linux-tutorial/assets/js/11.b9b41530.js"><link rel="prefetch" href="/linux-tutorial/assets/js/12.70a5dba8.js"><link rel="prefetch" href="/linux-tutorial/assets/js/13.857dcc43.js"><link rel="prefetch" href="/linux-tutorial/assets/js/14.5a603a55.js"><link rel="prefetch" href="/linux-tutorial/assets/js/15.d217acb7.js"><link rel="prefetch" href="/linux-tutorial/assets/js/16.ad565eae.js"><link rel="prefetch" href="/linux-tutorial/assets/js/17.d43e9f56.js"><link rel="prefetch" href="/linux-tutorial/assets/js/18.aa00ff43.js"><link rel="prefetch" href="/linux-tutorial/assets/js/19.43ce44b3.js"><link rel="prefetch" href="/linux-tutorial/assets/js/20.5618e1ff.js"><link rel="prefetch" href="/linux-tutorial/assets/js/21.1c5a41d7.js"><link rel="prefetch" href="/linux-tutorial/assets/js/22.fbe9fdf1.js"><link rel="prefetch" href="/linux-tutorial/assets/js/23.a4fb0e74.js"><link rel="prefetch" href="/linux-tutorial/assets/js/24.e3a23b69.js"><link rel="prefetch" href="/linux-tutorial/assets/js/25.9896afe9.js"><link rel="prefetch" href="/linux-tutorial/assets/js/26.96164082.js"><link rel="prefetch" href="/linux-tutorial/assets/js/27.391033bb.js"><link rel="prefetch" href="/linux-tutorial/assets/js/28.703f74c2.js"><link rel="prefetch" href="/linux-tutorial/assets/js/29.02a952cb.js"><link rel="prefetch" href="/linux-tutorial/assets/js/30.7e13628f.js"><link rel="prefetch" href="/linux-tutorial/assets/js/31.c4652f75.js"><link rel="prefetch" href="/linux-tutorial/assets/js/32.05d2cbec.js"><link rel="prefetch" href="/linux-tutorial/assets/js/33.3b265df8.js"><link rel="prefetch" href="/linux-tutorial/assets/js/34.26330a03.js"><link rel="prefetch" href="/linux-tutorial/assets/js/35.417d706d.js"><link rel="prefetch" href="/linux-tutorial/assets/js/36.0ed775e0.js"><link rel="prefetch" href="/linux-tutorial/assets/js/37.34430c74.js"><link rel="prefetch" href="/linux-tutorial/assets/js/38.87d5e0ff.js"><link rel="prefetch" href="/linux-tutorial/assets/js/39.7b648b3e.js"><link rel="prefetch" href="/linux-tutorial/assets/js/40.3b7a219e.js"><link rel="prefetch" href="/linux-tutorial/assets/js/41.e727eee9.js"><link rel="prefetch" href="/linux-tutorial/assets/js/42.0134c187.js"><link rel="prefetch" href="/linux-tutorial/assets/js/43.175e982f.js"><link rel="prefetch" href="/linux-tutorial/assets/js/44.72d90888.js"><link rel="prefetch" href="/linux-tutorial/assets/js/45.d49955bd.js"><link rel="prefetch" href="/linux-tutorial/assets/js/46.a9c290ec.js"><link rel="prefetch" href="/linux-tutorial/assets/js/47.cc639f04.js"><link rel="prefetch" href="/linux-tutorial/assets/js/48.98c78321.js"><link rel="prefetch" href="/linux-tutorial/assets/js/49.a7c3afed.js"><link rel="prefetch" href="/linux-tutorial/assets/js/50.22d8c542.js"><link rel="prefetch" href="/linux-tutorial/assets/js/51.28055fcd.js"><link rel="prefetch" href="/linux-tutorial/assets/js/52.f8103df5.js"><link rel="prefetch" href="/linux-tutorial/assets/js/54.e78d2776.js"><link rel="prefetch" href="/linux-tutorial/assets/js/55.3ce3079c.js"><link rel="prefetch" href="/linux-tutorial/assets/js/56.832958c9.js"><link rel="prefetch" href="/linux-tutorial/assets/js/57.961ce896.js"><link rel="prefetch" href="/linux-tutorial/assets/js/58.6d6fbc82.js"><link rel="prefetch" href="/linux-tutorial/assets/js/59.d5e48112.js"><link rel="prefetch" href="/linux-tutorial/assets/js/6.c8f4721c.js"><link rel="prefetch" href="/linux-tutorial/assets/js/60.7927b23b.js"><link rel="prefetch" href="/linux-tutorial/assets/js/61.ee233f24.js"><link rel="prefetch" href="/linux-tutorial/assets/js/62.6ba50cc7.js"><link rel="prefetch" href="/l
|
|||
|
|
<link rel="stylesheet" href="/linux-tutorial/assets/css/0.styles.45d9d031.css">
|
|||
|
|
</head>
|
|||
|
|
<body>
|
|||
|
|
<div id="app" data-server-rendered="true"><div class="theme-container"><header class="navbar"><div class="sidebar-button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" role="img" viewBox="0 0 448 512" class="icon"><path fill="currentColor" d="M436 124H12c-6.627 0-12-5.373-12-12V80c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12zm0 160H12c-6.627 0-12-5.373-12-12v-32c0-6.627 5.373-12 12-12h424c6.627 0 12 5.373 12 12v32c0 6.627-5.373 12-12 12z"></path></svg></div> <a href="/linux-tutorial/" class="home-link router-link-active"><img src="images/dunwu-logo-100.png" alt="LINUX-TUTORIAL" class="logo"> <span class="site-name can-hide">LINUX-TUTORIAL</span></a> <div class="links"><div class="search-box"><input aria-label="Search" autocomplete="off" spellcheck="false" value=""> <!----></div> <nav class="nav-links can-hide"><div class="nav-item"><a href="/linux-tutorial/linux/cli/" class="nav-link">
|
|||
|
|
Linux 命令
|
|||
|
|
</a></div><div class="nav-item"><a href="/linux-tutorial/linux/ops/" class="nav-link">
|
|||
|
|
Linux 运维
|
|||
|
|
</a></div><div class="nav-item"><a href="/linux-tutorial/linux/soft/" class="nav-link router-link-active">
|
|||
|
|
Linux 软件运维
|
|||
|
|
</a></div><div class="nav-item"><a href="/linux-tutorial/docker/" class="nav-link">
|
|||
|
|
Docker 教程
|
|||
|
|
</a></div><div class="nav-item"><a href="https://github.com/dunwu/blog" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
|||
|
|
🎯 博客
|
|||
|
|
<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></div> <a href="https://github.com/dunwu/linux-tutorial" target="_blank" rel="noopener noreferrer" class="repo-link">
|
|||
|
|
Github
|
|||
|
|
<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></nav></div></header> <div class="sidebar-mask"></div> <aside class="sidebar"><nav class="nav-links"><div class="nav-item"><a href="/linux-tutorial/linux/cli/" class="nav-link">
|
|||
|
|
Linux 命令
|
|||
|
|
</a></div><div class="nav-item"><a href="/linux-tutorial/linux/ops/" class="nav-link">
|
|||
|
|
Linux 运维
|
|||
|
|
</a></div><div class="nav-item"><a href="/linux-tutorial/linux/soft/" class="nav-link router-link-active">
|
|||
|
|
Linux 软件运维
|
|||
|
|
</a></div><div class="nav-item"><a href="/linux-tutorial/docker/" class="nav-link">
|
|||
|
|
Docker 教程
|
|||
|
|
</a></div><div class="nav-item"><a href="https://github.com/dunwu/blog" target="_blank" rel="noopener noreferrer" class="nav-link external">
|
|||
|
|
🎯 博客
|
|||
|
|
<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></div> <a href="https://github.com/dunwu/linux-tutorial" target="_blank" rel="noopener noreferrer" class="repo-link">
|
|||
|
|
Github
|
|||
|
|
<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></nav> <ul class="sidebar-links"><li><section class="sidebar-group depth-0"><p class="sidebar-heading open"><span>Elastic 技术栈之 Logstash 基础</span> <!----></p> <ul class="sidebar-links sidebar-group-items"><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#简介" class="sidebar-link">简介</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#功能" class="sidebar-link">功能</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#工作原理" class="sidebar-link">工作原理</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#设置" class="sidebar-link">设置</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#设置文件" class="sidebar-link">设置文件</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#logstash-yml-设置项" class="sidebar-link">logstash.yml 设置项</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#启动" class="sidebar-link">启动</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#命令行" class="sidebar-link">命令行</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#配置文件" class="sidebar-link">配置文件</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#插件" class="sidebar-link">插件</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#input" class="sidebar-link">input</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#filter" class="sidebar-link">filter</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#output" class="sidebar-link">output</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#codec" class="sidebar-link">codec</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#实战" class="sidebar-link">实战</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#传输控制台数据" class="sidebar-link">传输控制台数据</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#传输-logback-日志" class="sidebar-link">传输 logback 日志</a></li><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#传输文件" class="sidebar-link">传输文件</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#小技巧" class="sidebar-link">小技巧</a><ul class="sidebar-sub-headers"><li class="sidebar-sub-header"><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#启动、终止应用" class="sidebar-link">启动、终止应用</a></li></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#资料" class="sidebar-link">资料</a><ul class="sidebar-sub-headers"></ul></li><li><a href="/linux-tutorial/linux/soft/elastic/elastic-logstash.html#推荐阅读" class="sidebar-link">推荐阅读</a><ul class="sidebar-s
|
|||
|
|
</code></pre></div><p>其中 [options] 是您可以指定用于控制 Logstash 执行的命令行标志。</p> <p>在命令行上设置的任何标志都会覆盖 Logstash 设置文件（<code>logstash.yml</code>）中的相应设置，但设置文件本身不会更改。</p> <blockquote><p><strong>注</strong></p> <p>虽然可以通过指定命令行参数的方式，来控制 logstash 的运行方式，但显然这么做很麻烦。</p> <p>建议通过指定配置文件的方式，来控制 logstash 运行，启动命令如下：</p> <div class="language- extra-class"><pre class="language-text"><code>bin/logstash -f logstash.conf
|
|||
|
|
</code></pre></div><p>若想了解更多的命令行参数细节，请参考：https://www.elastic.co/guide/en/logstash/current/running-logstash-command-line.html</p></blockquote> <h3 id="配置文件"><a href="#配置文件" class="header-anchor">#</a> 配置文件</h3> <p>上节，我们了解到，logstash 可以执行 <code>bin/logstash -f logstash.conf</code> ，按照配置文件中的参数去覆盖默认设置文件（<code>logstash.yml</code>）中的设置。</p> <p>这节，我们就来学习一下这个配置文件如何配置参数。</p> <h4 id="配置文件结构"><a href="#配置文件结构" class="header-anchor">#</a> 配置文件结构</h4> <p>在工作原理一节中，我们已经知道了 Logstash 主要有三个工作阶段 input 、filter、output。而 logstash 配置文件文件结构也与之相对应：</p> <div class="language- extra-class"><pre class="language-text"><code>input {}
|
|||
|
|
|
|||
|
|
filter {}
|
|||
|
|
|
|||
|
|
output {}
|
|||
|
|
</code></pre></div><blockquote><p>每个部分都包含一个或多个插件的配置选项。如果指定了多个过滤器，则会按照它们在配置文件中的显示顺序应用它们。</p></blockquote> <h4 id="插件配置"><a href="#插件配置" class="header-anchor">#</a> 插件配置</h4> <p>插件的配置由插件名称和插件的一个设置块组成。</p> <p>下面的例子中配置了两个输入文件配置：</p> <div class="language- extra-class"><pre class="language-text"><code>input {
|
|||
|
|
file {
|
|||
|
|
path => "/var/log/messages"
|
|||
|
|
type => "syslog"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
file {
|
|||
|
|
path => "/var/log/apache/access.log"
|
|||
|
|
type => "apache"
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
</code></pre></div><p>您可以配置的设置因插件类型而异。你可以参考： <a href="https://www.elastic.co/guide/en/logstash/current/input-plugins.html" target="_blank" rel="noopener noreferrer">Input Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a>, <a href="https://www.elastic.co/guide/en/logstash/current/output-plugins.html" target="_blank" rel="noopener noreferrer">Output Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a>, <a href="https://www.elastic.co/guide/en/logstash/current/filter-plugins.html" target="_blank" rel="noopener noreferrer">Filter Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a>, 和 <a href="https://www.elastic.co/guide/en/logstash/current/codec-plugins.html" target="_blank" rel="noopener noreferrer">Codec Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 。</p> <h4 id="值类型"><a href="#值类型" class="header-anchor">#</a> 值类型</h4> <p>一个插件可以要求设置的值是一个特定的类型，比如布尔值，列表或哈希值。以下值类型受支持。</p> <ul><li>Array</li></ul> <div class="language- extra-class"><pre class="language-text"><code> users => [ {id => 1, name => bob}, {id => 2, name => jane} ]
|
|||
|
|
</code></pre></div><ul><li>Lists</li></ul> <div class="language- extra-class"><pre class="language-text"><code> path => [ "/var/log/messages", "/var/log/*.log" ]
|
|||
|
|
uris => [ "http://elastic.co", "http://example.net" ]
|
|||
|
|
</code></pre></div><ul><li>Boolean</li></ul> <div class="language- extra-class"><pre class="language-text"><code> ssl_enable => true
|
|||
|
|
</code></pre></div><ul><li>Bytes</li></ul> <div class="language- extra-class"><pre class="language-text"><code> my_bytes => "1113" # 1113 bytes
|
|||
|
|
my_bytes => "10MiB" # 10485760 bytes
|
|||
|
|
my_bytes => "100kib" # 102400 bytes
|
|||
|
|
my_bytes => "180 mb" # 180000000 bytes
|
|||
|
|
</code></pre></div><ul><li>Codec</li></ul> <div class="language- extra-class"><pre class="language-text"><code> codec => "json"
|
|||
|
|
</code></pre></div><ul><li>Hash</li></ul> <div class="language- extra-class"><pre class="language-text"><code>match => {
|
|||
|
|
"field1" => "value1"
|
|||
|
|
"field2" => "value2"
|
|||
|
|
...
|
|||
|
|
}
|
|||
|
|
</code></pre></div><ul><li>Number</li></ul> <div class="language- extra-class"><pre class="language-text"><code> port => 33
|
|||
|
|
</code></pre></div><ul><li>Password</li></ul> <div class="language- extra-class"><pre class="language-text"><code> my_password => "password"
|
|||
|
|
</code></pre></div><ul><li>URI</li></ul> <div class="language- extra-class"><pre class="language-text"><code> my_uri => "http://foo:bar@example.net"
|
|||
|
|
</code></pre></div><ul><li>Path</li></ul> <div class="language- extra-class"><pre class="language-text"><code> my_path => "/tmp/logstash"
|
|||
|
|
</code></pre></div><ul><li><p>String</p></li> <li><p>转义字符</p></li></ul> <h2 id="插件"><a href="#插件" class="header-anchor">#</a> 插件</h2> <h3 id="input"><a href="#input" class="header-anchor">#</a> input</h3> <blockquote><p>Logstash 支持各种输入选择 ，可以在同一时间从众多常用来源捕捉事件。能够以连续的流式传输方式，轻松地从您的日志、指标、Web 应用、数据存储以及各种 AWS 服务采集数据。</p></blockquote> <h4 id="常用-input-插件"><a href="#常用-input-插件" class="header-anchor">#</a> 常用 input 插件</h4> <ul><li><strong>file</strong>：从文件系统上的文件读取，就像UNIX命令 <code>tail -0F</code> 一样</li> <li>**syslog：**在众所周知的端口514上侦听系统日志消息，并根据RFC3164格式进行解析</li> <li>**redis：**从redis服务器读取，使用redis通道和redis列表。 Redis经常用作集中式Logstash安装中的“代理”，它将来自远程Logstash“托运人”的Logstash事件排队。</li> <li>**beats：**处理由Filebeat发送的事件。</li></ul> <p>更多详情请见：<a href="https://www.elastic.co/guide/en/logstash/current/input-plugins.html" target="_blank" rel="noopener noreferrer">Input Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p> <h3 id="filter"><a href="#filter" class="header-anchor">#</a> filter</h3> <blockquote><p>过滤器是Logstash管道中的中间处理设备。如果符合特定条件，您可以将条件过滤器组合在一起，对事件执行操作。</p></blockquote> <h4 id="常用-filter-插件"><a href="#常用-filter-插件" class="header-anchor">#</a> 常用 filter 插件</h4> <ul><li><p>**grok：**解析和结构任意文本。 Grok目前是Logstash中将非结构化日志数据解析为结构化和可查询的最佳方法。</p></li> <li><p>**mutate：**对事件字段执行一般转换。您可以重命名，删除，替换和修改事件中的字段。</p></li> <li><p>**drop：**完全放弃一个事件，例如调试事件。</p></li> <li><p>**clone：**制作一个事件的副本，可能会添加或删除字段。</p></li> <li><p>**geoip：**添加有关IP地址的地理位置的信息（也可以在Kibana中显示惊人的图表！）</p></li></ul> <p>更多详情请见：<a href="https://www.elastic.co/guide/en/logstash/current/filter-plugins.html" target="_blank" rel="noopener noreferrer">Filter Plugins<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p> <h3 id="output"><a href="#output" class="header-anchor">#</a> output</h3> <blockquote><p>输出是Logstash管道的最后阶段。一个事件可以通过多个输出，但是一旦所有输出处理完成，事件就完成了执行。</p></blockquote> <h4 id="常用-output-插件"><a href="#常用-output-插件" class="header-anchor">#</a> 常用 output 插件</h4> <ul><li>**elasticsearch：**将事件数据发送给 Elasticsearch（推荐模式）。</li> <li>**file：**将事件数据写入文件或磁盘。</li> <li>**graphite：**将事件数据发送给 graphite（一个流行的开源工具，存储和绘制指标。 http://graphite.r
|
|||
|
|
output {
|
|||
|
|
elasticsearch { hosts => ["localhost:9200"] }
|
|||
|
|
stdout { codec => rubydebug }
|
|||
|
|
}
|
|||
|
|
</code></pre></div><p>更多配置项可以参考：https://www.elastic.co/guide/en/logstash/current/plugins-inputs-stdin.html</p> <p>（2）执行 logstash，使用 <code>-f</code> 来指定你的配置文件：</p> <div class="language- extra-class"><pre class="language-text"><code>bin/logstash -f logstash-input-stdin.conf
|
|||
|
|
</code></pre></div><h3 id="传输-logback-日志"><a href="#传输-logback-日志" class="header-anchor">#</a> 传输 logback 日志</h3> <blockquote><p>elk 默认使用的 Java 日志工具是 log4j2 ，并不支持 logback 和 log4j。</p> <p>想使用 logback + logstash ，可以使用 <a href="https://github.com/logstash/logstash-logback-encoder" target="_blank" rel="noopener noreferrer">logstash-logback-encoder<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 。<a href="https://github.com/logstash/logstash-logback-encoder" target="_blank" rel="noopener noreferrer">logstash-logback-encoder<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 提供了 UDP / TCP / 异步方式来传输日志数据到 logstash。</p> <p>如果你使用的是 log4j ，也不是不可以用这种方式，只要引入桥接 jar 包即可。如果你对 log4j 、logback ，或是桥接 jar 包不太了解，可以参考我的这篇博文：<a href="https://github.com/dunwu/JavaStack/blob/master/docs/javalib/java-log.md" target="_blank" rel="noopener noreferrer">细说 Java 主流日志工具库<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 。</p></blockquote> <h4 id="tcp-应用"><a href="#tcp-应用" class="header-anchor">#</a> TCP 应用</h4> <ol><li><p>logstash 配置</p> <p>（1）创建 <code>logstash-input-tcp.conf</code> ：</p></li></ol> <div class="language- extra-class"><pre class="language-text"><code>input {
|
|||
|
|
tcp {
|
|||
|
|
port => 9251
|
|||
|
|
codec => json_lines
|
|||
|
|
mode => server
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
output {
|
|||
|
|
elasticsearch { hosts => ["localhost:9200"] }
|
|||
|
|
stdout { codec => rubydebug }
|
|||
|
|
}
|
|||
|
|
</code></pre></div><p>更多配置项可以参考：https://www.elastic.co/guide/en/logstash/current/plugins-inputs-tcp.html</p> <p>（2）执行 logstash，使用 <code>-f</code> 来指定你的配置文件：<code>bin/logstash -f logstash-input-udp.conf</code></p> <ol start="2"><li><p>java 应用配置</p> <p>（1）在 Java 应用的 pom.xml 中引入 jar 包：</p></li></ol> <div class="language-xml extra-class"><pre class="language-xml"><code><span class="token tag"><span class="token tag"><span class="token punctuation"><</span>dependency</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>groupId</span><span class="token punctuation">></span></span>net.logstash.logback<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>groupId</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>artifactId</span><span class="token punctuation">></span></span>logstash-logback-encoder<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>artifactId</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>version</span><span class="token punctuation">></span></span>4.11<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>version</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>dependency</span><span class="token punctuation">></span></span>
|
|||
|
|
|
|||
|
|
<span class="token comment"><!-- logback 依赖包 --></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>dependency</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>groupId</span><span class="token punctuation">></span></span>ch.qos.logback<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>groupId</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>artifactId</span><span class="token punctuation">></span></span>logback-core<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>artifactId</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>version</span><span class="token punctuation">></span></span>1.2.3<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>version</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>dependency</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>dependency</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>groupId</span><span class="token punctuation">></span></span>ch.qos.logback<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>groupId</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>artifactId</span><span class="token punctuation">></span></span>logback-classic<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>artifactId</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>version</span><span class="token punctuation">></span></span>1.2.3<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>version</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>dependency</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>dependency</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>groupId</span><span class="token punctuation">></span></span>ch.qos.logback<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>groupId</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>artifactId</span><span class="token punctuation">></span></span>logback-access<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>artifactId</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>version</span><span class="token punctuation">></span></span>1.2.3<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>version</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>dependency</span><span class="token punctuation">></span></span>
|
|||
|
|
</code></pre></div><p>（2）接着，在 logback.xml 中添加 appender</p> <div class="language-xml extra-class"><pre class="language-xml"><code><span class="token tag"><span class="token tag"><span class="token punctuation"><</span>appender</span> <span class="token attr-name">name</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>ELK-TCP<span class="token punctuation">"</span></span> <span class="token attr-name">class</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>net.logstash.logback.appender.LogstashTcpSocketAppender<span class="token punctuation">"</span></span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token comment"><!--
|
|||
|
|
destination 是 logstash 服务的 host:port，
|
|||
|
|
相当于和 logstash 建立了管道，将日志数据定向传输到 logstash
|
|||
|
|
--></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>destination</span><span class="token punctuation">></span></span>192.168.28.32:9251<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>destination</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>encoder</span> <span class="token attr-name">charset</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>UTF-8<span class="token punctuation">"</span></span> <span class="token attr-name">class</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>net.logstash.logback.encoder.LogstashEncoder<span class="token punctuation">"</span></span><span class="token punctuation">/></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>appender</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>logger</span> <span class="token attr-name">name</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>io.github.dunwu.spring<span class="token punctuation">"</span></span> <span class="token attr-name">level</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>TRACE<span class="token punctuation">"</span></span> <span class="token attr-name">additivity</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>false<span class="token punctuation">"</span></span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>appender-ref</span> <span class="token attr-name">ref</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>ELK-TCP<span class="token punctuation">"</span></span> <span class="token punctuation">/></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>logger</span><span class="token punctuation">></span></span>
|
|||
|
|
</code></pre></div><p>（3）接下来，就是 logback 的具体使用 ，如果对此不了解，不妨参考一下我的这篇博文：<a href="https://github.com/dunwu/JavaStack/blob/master/docs/javalib/java-log.md" target="_blank" rel="noopener noreferrer">细说 Java 主流日志工具库<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 。</p> <p><strong>实例：</strong><a href="https://github.com/dunwu/JavaStack/blob/master/codes/javatool/src/main/resources/logback.xml" target="_blank" rel="noopener noreferrer">我的logback.xml<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p> <h4 id="udp-应用"><a href="#udp-应用" class="header-anchor">#</a> UDP 应用</h4> <p>UDP 和 TCP 的使用方式大同小异。</p> <ol><li><p>logstash 配置</p> <p>（1）创建 <code>logstash-input-udp.conf</code> ：</p></li></ol> <div class="language- extra-class"><pre class="language-text"><code>input {
|
|||
|
|
udp {
|
|||
|
|
port => 9250
|
|||
|
|
codec => json
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
output {
|
|||
|
|
elasticsearch { hosts => ["localhost:9200"] }
|
|||
|
|
stdout { codec => rubydebug }
|
|||
|
|
}
|
|||
|
|
</code></pre></div><p>更多配置项可以参考：https://www.elastic.co/guide/en/logstash/current/plugins-inputs-udp.html</p> <p>（2）执行 logstash，使用 <code>-f</code> 来指定你的配置文件：<code>bin/logstash -f logstash-input-udp.conf</code></p> <ol start="2"><li><p>java 应用配置</p> <p>（1）在 Java 应用的 pom.xml 中引入 jar 包：</p> <p>与 <strong>TCP 应用</strong> 一节中的引入依赖包完全相同。</p> <p>（2）接着，在 logback.xml 中添加 appender</p></li></ol> <div class="language-xml extra-class"><pre class="language-xml"><code><span class="token tag"><span class="token tag"><span class="token punctuation"><</span>appender</span> <span class="token attr-name">name</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>ELK-UDP<span class="token punctuation">"</span></span> <span class="token attr-name">class</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>net.logstash.logback.appender.LogstashSocketAppender<span class="token punctuation">"</span></span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>host</span><span class="token punctuation">></span></span>192.168.28.32<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>host</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>port</span><span class="token punctuation">></span></span>9250<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>port</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>appender</span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>logger</span> <span class="token attr-name">name</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>io.github.dunwu.spring<span class="token punctuation">"</span></span> <span class="token attr-name">level</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>TRACE<span class="token punctuation">"</span></span> <span class="token attr-name">additivity</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>false<span class="token punctuation">"</span></span><span class="token punctuation">></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"><</span>appender-ref</span> <span class="token attr-name">ref</span><span class="token attr-value"><span class="token punctuation attr-equals">=</span><span class="token punctuation">"</span>ELK-UDP<span class="token punctuation">"</span></span> <span class="token punctuation">/></span></span>
|
|||
|
|
<span class="token tag"><span class="token tag"><span class="token punctuation"></</span>logger</span><span class="token punctuation">></span></span>
|
|||
|
|
</code></pre></div><p>（3）接下来，就是 logback 的具体使用 ，如果对此不了解，不妨参考一下我的这篇博文：<a href="https://github.com/dunwu/JavaStack/blob/master/docs/javalib/java-log.md" target="_blank" rel="noopener noreferrer">细说 Java 主流日志工具库<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a> 。</p> <p><strong>实例：</strong><a href="https://github.com/dunwu/JavaStack/blob/master/codes/javatool/src/main/resources/logback.xml" target="_blank" rel="noopener noreferrer">我的logback.xml<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></p> <h3 id="传输文件"><a href="#传输文件" class="header-anchor">#</a> 传输文件</h3> <blockquote><p>在 Java Web 领域，需要用到一些重要的工具，例如 Tomcat 、Nginx 、Mysql 等。这些不属于业务应用，但是它们的日志数据对于定位问题、分析统计同样很重要。这时无法使用 logback 方式将它们的日志传输到 logstash。</p> <p>如何采集这些日志文件呢？别急，你可以使用 logstash 的 file input 插件。</p> <p>需要注意的是，传输文件这种方式，必须在日志所在的机器上部署 logstash 。</p></blockquote> <p><strong>应用</strong></p> <p>logstash 配置</p> <p>（1）创建 <code>logstash-input-file.conf</code> ：</p> <div class="language- extra-class"><pre class="language-text"><code>input {
|
|||
|
|
file {
|
|||
|
|
path => ["/var/log/nginx/access.log"]
|
|||
|
|
type => "nginx-access-log"
|
|||
|
|
start_position => "beginning"
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
output {
|
|||
|
|
if [type] == "nginx-access-log" {
|
|||
|
|
elasticsearch {
|
|||
|
|
hosts => ["localhost:9200"]
|
|||
|
|
index => "nginx-access-log"
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
</code></pre></div><p>（2）执行 logstash，使用 <code>-f</code> 来指定你的配置文件：<code>bin/logstash -f logstash-input-file.conf</code></p> <p>更多配置项可以参考：https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html</p> <h2 id="小技巧"><a href="#小技巧" class="header-anchor">#</a> 小技巧</h2> <h3 id="启动、终止应用"><a href="#启动、终止应用" class="header-anchor">#</a> 启动、终止应用</h3> <p>如果你的 logstash 每次都是通过指定配置文件方式启动。不妨建立一个启动脚本。</p> <div class="language- extra-class"><pre class="language-text"><code># cd xxx 进入 logstash 安装目录下的 bin 目录
|
|||
|
|
logstash -f logstash.conf
|
|||
|
|
</code></pre></div><p>如果你的 logstash 运行在 linux 系统下，不妨使用 nohup 来启动一个守护进程。这样做的好处在于，即使关闭终端，应用仍会运行。</p> <p><strong>创建 startup.sh</strong></p> <div class="language- extra-class"><pre class="language-text"><code>nohup ./logstash -f logstash.conf >> nohup.out 2>&1 &
|
|||
|
|
</code></pre></div><p>终止应用没有什么好方法，你只能使用 ps -ef | grep logstash ，查出进程，将其kill 。不过，我们可以写一个脚本来干这件事：</p> <p><strong>创建 shutdown.sh</strong></p> <p>脚本不多解释，请自行领会作用。</p> <div class="language- extra-class"><pre class="language-text"><code>PID=`ps -ef | grep logstash | awk '{ print $2}' | head -n 1`
|
|||
|
|
kill -9 ${PID}
|
|||
|
|
</code></pre></div><h2 id="资料"><a href="#资料" class="header-anchor">#</a> 资料</h2> <ul><li><a href="https://www.elastic.co/guide/en/logstash/current/index.html" target="_blank" rel="noopener noreferrer">Logstash 官方文档<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li> <li><a href="https://github.com/logstash/logstash-logback-encoder" target="_blank" rel="noopener noreferrer">logstash-logback-encoder<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li> <li><a href="https://github.com/chenryn/logstash-best-practice-cn" target="_blank" rel="noopener noreferrer">ELK Stack权威指南<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li> <li><a href="https://github.com/judasn/Linux-Tutorial/blob/master/ELK-Install-And-Settings.md" target="_blank" rel="noopener noreferrer">ELK（Elasticsearch、Logstash、Kibana）安装和配置<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li></ul> <h2 id="推荐阅读"><a href="#推荐阅读" class="header-anchor">#</a> 推荐阅读</h2> <ul><li><a href="https://github.com/dunwu/JavaStack/blob/master/docs/javatool/elastic/README.md" target="_blank" rel="noopener noreferrer">Elastic 技术栈<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.2,37.2 85.2,14.9 62.8,14.9 62.8,22.9 71.5,22.9"></polygon></svg> <span class="sr-only">(opens new window)</span></span></a></li> <li><a href="https://github.com/dunwu/JavaStack" target="_blank" rel="noopener noreferrer">JavaStack<span><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" focusable="false" x="0px" y="0px" viewBox="0 0 100 100" width="15" height="15" class="icon outbound"><path fill="currentColor" d="M18.8,85.1h56l0,0c2.2,0,4-1.8,4-4v-32h-8v28h-48v-48h28v-8h-32l0,0c-2.2,0-4,1.8-4,4v56C14.8,83.3,16.6,85.1,18.8,85.1z"></path> <polygon fill="currentColor" points="45.7,48.7 51.3,54.3 77.2,28.5 77.2,37.2 85.
|
|||
|
|
<script src="/linux-tutorial/assets/js/app.79a38eea.js" defer></script><script src="/linux-tutorial/assets/js/4.fb6e0f89.js" defer></script><script src="/linux-tutorial/assets/js/53.76541550.js" defer></script><script src="/linux-tutorial/assets/js/5.cb43ecfb.js" defer></script>
|
|||
|
|
</body>
|
|||
|
|
</html>
|