{"id":717,"date":"2021-04-06T15:57:51","date_gmt":"2021-04-06T07:57:51","guid":{"rendered":"https:\/\/www.yinyubo.cn\/?p=717"},"modified":"2022-05-17T09:29:27","modified_gmt":"2022-05-17T01:29:27","slug":"datastreamslogstashind","status":"publish","type":"post","link":"https:\/\/www.yinyubo.com\/?p=717","title":{"rendered":"DataStreams+logstash+ILM\u8fdb\u884c\u65e5\u5fd7\u5b9a\u65f6\u5220\u9664\uff0c\u8282\u7701\u786c\u76d8\u8d44\u6e90"},"content":{"rendered":"<h1 id=\"id-\u901a\u8fc7\u751f\u547d\u5468\u671f\u7ba1\u7406\u5de5\u5177\uff0c\u8282\u7701ELK\u786c\u76d8\u8d44\u6e90-\u80cc\u666f\">\u80cc\u666f<\/h1>\n<p>\u76ee\u524d\u6240\u6709\u7684K8S\u4e0a\u7684\u5bb9\u5668\u65e5\u5fd7\u90fd\u88ab\u6536\u96c6\u5230\u4e86\u6211\u4eec\u7684ELK\u4e0a\uff0c\u968f\u7740\u65f6\u95f4\u7684\u63a8\u79fb\uff0cELK\u4e0a\u7684\u65e5\u5fd7\u6240\u5360\u7684\u5b58\u50a8\u7a7a\u95f4\u8d8a\u6765\u8d8a\u591a\uff0c\u6211\u4eec\u9700\u8981\u4e00\u4e2a\u5b9a\u65f6\u6e05\u7406\u7684\u7b56\u7565\uff0c\u4ee5\u8282\u7ea6\u786c\u76d8\u8d44\u6e90\u3002<br \/>\n\u6211\u4eec\u4e3b\u8981\u914d\u7f6e\u4ee5\u4e0bELK\u91cc\u7684\u8fd9\u51e0\u4e2a\u5730\u65b9<\/p>\n<ul>\n<li>\u901a\u8fc7kibana\u65b0\u589e\u4e00\u4e2alifecycle policies<\/li>\n<li>\u901a\u8fc7kibana\u65b0\u589e\u4e00\u4e2aindex template,\u6ce8\u610f\u914d\u7f6eDataStreams<\/li>\n<li>logstash \u7684logstashPipeline<\/li>\n<li>filebeat\u7684filebeat.yml\u6587\u4ef6<\/li>\n<\/ul>\n<hr \/>\n<h1 id=\"id-\u901a\u8fc7\u751f\u547d\u5468\u671f\u7ba1\u7406\u5de5\u5177\uff0c\u8282\u7701ELK\u786c\u76d8\u8d44\u6e90-\u7b80\u8981\u914d\u7f6e\u56fe\">\u7b80\u8981\u914d\u7f6e\u56fe<\/h1>\n<p><img loading=\"lazy\" decoding=\"async\" class=\"alignnone wp-image-718 size-large\" src=\"https:\/\/yinyubo-1257235934.cos.ap-nanjing.myqcloud.com\/content\/Image-2-8-1024x314.png\" alt=\"\" width=\"900\" height=\"276\" \/><\/p>\n<hr \/>\n<h1 id=\"id-\u901a\u8fc7\u751f\u547d\u5468\u671f\u7ba1\u7406\u5de5\u5177\uff0c\u8282\u7701ELK\u786c\u76d8\u8d44\u6e90-Kibana\u4e0a\u6dfb\u52a0LifecyclePolicies\">Kibana\u4e0a\u6dfb\u52a0\u00a0Lifecycle Policies<\/h1>\n<p>1.\u70b9\u51fb\u83dc\u5355\u680f\u7684\u3010management\u3011-&gt;\u70b9\u51fb\u3010stack management\u3011<br \/>\n2.\u70b9\u51fbDATA\u76ee\u5f55\u4e0b\u7684\u3010Index Lifecycle Policies\u3011<br \/>\n3.\u70b9\u51fb\u3010Create policy\u3011\u521b\u5efa\u4e00\u4e2a\u65b0\u7684\u751f\u547d\u5468\u671f\u89c4\u5219<br \/>\n4.\u6d4b\u8bd5\u7684\u8bdd\uff0c\u89c4\u5219\u5c31\u968f\u4fbf\u914d\u7f6e\u4e00\u4e2a\u6bcf10\u5206\u949f\u8fed\u4ee3\u4e00\u4e2a\u65b0\u7684\uff0c\u5220\u9664\u8d85\u8fc71\u5c0f\u65f6\u7684index<br \/>\n\u671f\u671b\u6548\u679c\uff1a<br \/>\n\u5bf9\u5e94\u7684index\u4f1a\u4ece00001\u5f00\u59cb\u6bcf\u969410\u5206\u949f\u5f80\u4e0a+1\uff0c\u540c\u65f6\u6700\u591a\u5b58\u57287\u4e2aindex\u3002<br \/>\n<img loading=\"lazy\" decoding=\"async\" class=\"alignnone wp-image-719 size-large\" src=\"https:\/\/yinyubo-1257235934.cos.ap-nanjing.myqcloud.com\/content\/Image-3-4-1024x639.png\" alt=\"\" width=\"900\" height=\"562\" \/><\/p>\n<hr \/>\n<h1 id=\"id-\u901a\u8fc7\u751f\u547d\u5468\u671f\u7ba1\u7406\u5de5\u5177\uff0c\u8282\u7701ELK\u786c\u76d8\u8d44\u6e90-Kibana\u4e0a\u6dfb\u52a0IndexTemplates\">Kibana\u4e0a\u6dfb\u52a0Index Templates<\/h1>\n<p>1.\u70b9\u51fb\u83dc\u5355\u680f\u7684\u3010management\u3011-&gt;\u70b9\u51fb\u3010stack management\u3011<br \/>\n2.\u70b9\u51fbDATA\u76ee\u5f55\u4e0b\u7684\u3010Index Management\u3011<br \/>\n3.\u70b9\u51fb\u3010Index Templates\u3011\u5c0f\u6807\u7b7e,\u3010Create template\u3011\u521b\u5efa\u6a21\u677f<br \/>\n4.index patterns\u5339\u914d\u6211\u4eeclogstash\u4e0a\u4f20\u6765\u7684index,\u6bd4\u5982192*<br \/>\n5.Data stream\u7684\u914d\u7f6e\u6309\u94ae\u6253\u5f00<br \/>\n<img loading=\"lazy\" decoding=\"async\" class=\"alignnone wp-image-720 size-large\" src=\"https:\/\/yinyubo-1257235934.cos.ap-nanjing.myqcloud.com\/content\/Image-4-2-1024x567.png\" alt=\"\" width=\"900\" height=\"498\" \/><br \/>\n6.index settings\u914d\u7f6e\u4e0a\u6211\u4eec\u4e0a\u4e00\u6b65\u6dfb\u52a0\u7684Lifecycle Policies<\/p>\n<div class=\"code panel pdl conf-macro output-block\" data-hasbody=\"true\" data-macro-name=\"code\">\n<div class=\"codeContent panelContent pdl\">\n<div>\n<div id=\"highlighter_821512\" class=\"syntaxhighlighter sh-confluence nogutter java\"><img loading=\"lazy\" decoding=\"async\" class=\"alignnone wp-image-721 size-large\" src=\"https:\/\/yinyubo-1257235934.cos.ap-nanjing.myqcloud.com\/content\/Image-5-2-1024x434.png\" alt=\"\" width=\"900\" height=\"381\" \/><\/div>\n<div>7.mappings\u53c2\u6570\u9700\u8981\u914d\u7f6e\u3010mapped fields\u3011\u548c\u3010Dynamic Template\u3011\u5185\u5bb9\u53ef\u4ee5\u4ecelogstash\u7684\u914d\u7f6e\u91cc\u5b8c\u6574\u590d\u5236\u8fc7\u6765<br \/>\n8.\u5176\u4ed6\u7684\u8bf8\u5982component telmpalte \u548cAliases\u90fd\u4e0d\u7528\u914d\u7f6e\u4e86\u3002\u4fdd\u5b58\u8fd9\u4e2aindex template\u5c31\u884c\u4e86<\/p>\n<hr \/>\n<h1 id=\"id-\u901a\u8fc7\u751f\u547d\u5468\u671f\u7ba1\u7406\u5de5\u5177\uff0c\u8282\u7701ELK\u786c\u76d8\u8d44\u6e90-logstash\u914d\u7f6e\">logstash\u914d\u7f6e<\/h1>\n<p>\u56e0\u4e3a\u8981\u52a8\u6001\u751f\u6210index\uff0c\u6240\u4ee5\u8981\u5199\u4e00\u4e9bfilter\u89c4\u5219,\u8fd9\u91cc\u5c31\u4e0d\u8d34\u51fa\u6765\u4e86\u3002\u5173\u952e\u6ce8\u610foutput\u91cc\u8981\u8bbe\u7f6e<\/p>\n<pre class=\"EnlighterJSRAW\" data-enlighter-language=\"golang\"><code class=\"EnlighterJSRAW\" data-enlighter-language=\"golang\">\"action\" =&gt; \"create\"\u548cilm_enabled =&gt; false<\/code><\/pre>\n<pre class=\"lang:yaml decode:true  EnlighterJSRAW\" data-enlighter-language=\"golang\"><code class=\"EnlighterJSRAW\" data-enlighter-language=\"golang\">logstash.conf: |\n    input {\n      beats {\n        port =&gt; 5044\n      }\n      tcp {\n        port =&gt; 9999\n      }\n      udp {\n        port =&gt; 9998\n      }\n    }\n    filter {\n      json {\n        source =&gt; \"message\"\n      }\n      if [app_name] {\n        mutate {\n          add_field =&gt; {\n            \"index_name\" =&gt; \"%{app_name}\"\n          }\n        }\n      } else {\n        mutate {\n          add_field =&gt; {\n            \"index_name\" =&gt;  \"non_index_log\"\n          }\n        }\n      }\n    }\n    output {\n        elasticsearch {\n           hosts =&gt; [\"http:\/\/elasticsearch-master-headless:9200\"]\n           index =&gt; \"%{index_name}\"\n           action =&gt; \"create\"\n           ilm_enabled =&gt; false\n        }\n        stdout { codec =&gt; rubydebug }\n    }<\/code><\/pre>\n<hr \/>\n<h1 id=\"id-\u901a\u8fc7\u751f\u547d\u5468\u671f\u7ba1\u7406\u5de5\u5177\uff0c\u8282\u7701ELK\u786c\u76d8\u8d44\u6e90-filebeat\u914d\u7f6e\">filebeat\u914d\u7f6e<\/h1>\n<p>\u56e0\u4e3a\u6709\u4e00\u4e9b\u65e5\u5fd7\u662f\u901a\u8fc7filebeat\u4f20\u4e0a\u6765\u7684\uff0c\u6240\u4ee5filebeat\u4e5f\u8981\u8fdb\u884c\u5c11\u91cf\u7684\u914d\u7f6e\uff0c\u4f20\u4e00\u4e2aapp_name\u5230logstash\u7528\u4e8e\u751f\u6210index<\/p>\n<pre class=\"lang:yaml decode:true  EnlighterJSRAW\" data-enlighter-language=\"golang\"><code class=\"EnlighterJSRAW\" data-enlighter-language=\"golang\">filebeat.inputs:\n  - type: log\n    paths:\n      - \"\/log\/*.log\"\nprocessors:\n  - decode_json_fields:\n        fields: [\"message\"]\n        process_array: false\n        max_depth: 1\n        target: \"\"\n        overwrite_keys: false\n  - add_fields:\n      target: ''\n      fields:\n        app_name: \"{{ .Values.nodeSelector.internet_ip}}-aimp-frontend-v2-ux\"\noutput.logstash:\n  hosts: [\"{{ .Values.logs.logstash.host }}:{{ .Values.logs.logstash.beatport }}\"]<\/code><\/pre>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>\u80cc\u666f \u76ee\u524d\u6240\u6709\u7684K8S\u4e0a\u7684\u5bb9\u5668\u65e5\u5fd7\u90fd\u88ab\u6536\u96c6\u5230\u4e86\u6211\u4eec\u7684ELK\u4e0a\uff0c\u968f\u7740\u65f6\u95f4\u7684\u63a8\u79fb\uff0cELK\u4e0a\u7684\u65e5\u5fd7\u6240\u5360\u7684\u5b58\u50a8\u7a7a\u95f4\u8d8a\u6765 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[2],"tags":[],"class_list":["post-717","post","type-post","status-publish","format-standard","hentry","category-elasticsearch"],"_links":{"self":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts\/717","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=717"}],"version-history":[{"count":2,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts\/717\/revisions"}],"predecessor-version":[{"id":854,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts\/717\/revisions\/854"}],"wp:attachment":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=717"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=717"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=717"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}