{"id":1034,"date":"2021-11-16T14:47:54","date_gmt":"2021-11-16T06:47:54","guid":{"rendered":"https:\/\/www.yinyubo.com\/?p=1034"},"modified":"2022-05-16T18:02:25","modified_gmt":"2022-05-16T10:02:25","slug":"fluentd","status":"publish","type":"post","link":"https:\/\/www.yinyubo.com\/?p=1034","title":{"rendered":"fluentd\u63a5\u5165Elasticsearch\u7684\u7b80\u5355\u4f8b\u5b50"},"content":{"rendered":"\n<h2 class=\"wp-block-heading\">\u80cc\u666f<\/h2>\n\n\n\n<p>\u6700\u8fd1\u60f3\u5b66\u4e60\u4e00\u4e0belasticsearch\u548cfluentd\u7684\u914d\u5408\u4f7f\u7528\uff0c fluentd\u6bd4logstash\u8282\u7701\u592a\u591a\u8d44\u6e90\u4e86\uff0c\u6240\u4ee5\u5c31\u6709\u4e86\u5982\u4e0b\u6587\u7ae0<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">Elasticsearch\u5feb\u6377\u5b89\u88c5(\u4f7f\u7528ECK\u65b9\u5f0f)<\/h2>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\"><p>\u53c2\u8003\u6587\u7ae0<\/p><cite>https:\/\/www.elastic.co\/guide\/en\/cloud-on-k8s\/1.8\/k8s-deploy-eck.html<\/cite><\/blockquote>\n\n\n\n<p>\u5148\u5b89\u88c5\u4e00\u4e2aeck\u7684operator<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>kubectl create -f https:\/\/download.elastic.co\/downloads\/eck\/1.8.0\/crds.yaml\nkubectl apply -f https:\/\/download.elastic.co\/downloads\/eck\/1.8.0\/operator.yaml<\/code><\/pre>\n\n\n\n<p>\u7b49\u547d\u4ee4\u4ecb\u7ecd\uff0c\u8f93\u5165\u4e0b\u9762\u547d\u4ee4\u67e5\u770b\u65e5\u5fd7<br><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>kubectl -n elastic-system logs -f statefulset.apps\/elastic-operator\n<\/code><\/pre>\n\n\n\n<p>\u5b89\u88c5elasticsearch<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>cat &lt;&lt;EOF | kubectl apply -f -\r\napiVersion: elasticsearch.k8s.elastic.co\/v1\r\nkind: Elasticsearch\r\nmetadata:\r\n  name: quickstart\r\nspec:\r\n  version: 7.15.2\r\n  nodeSets:\r\n  - name: default\r\n    count: 1\r\n    config:\r\n      node.store.allow_mmap: false\r\nEOF<\/code><\/pre>\n\n\n\n<p>\u5b89\u88c5\u5b8c\u6210\u540e\uff0c\u8f93\u5165\u547d\u4ee4\uff0c\u83b7\u5f97es\u7684\u5bc6\u7801\uff0c\u9ed8\u8ba4\u8d26\u6237\u662felastic<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>PASSWORD=$(kubectl get secret quickstart-es-elastic-user -o go-template='{{.data.elastic | base64decode}}')\r\n<\/code><\/pre>\n\n\n\n<p>\u90e8\u7f72\u5b8c\u6bd5\u540e\uff0c\u53ef\u4ee5\u901a\u8fc7port-forward\u8f6c\u53d1elasticsearch\u7684\u7aef\u53e3\u5230\u5916\u90e8\u8fdb\u884c\u6d4b\u8bd5<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>kubectl port-forward service\/quickstart-es-http 9200\n\r<\/code><\/pre>\n\n\n\n<p>\u518d\u5b89\u88c5\u4e00\u4e2akibana<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>cat &lt;&lt;EOF | kubectl apply -f -\r\napiVersion: kibana.k8s.elastic.co\/v1\r\nkind: Kibana\r\nmetadata:\r\n  name: quickstart\r\nspec:\r\n  version: 7.15.2\r\n  count: 1\r\n  elasticsearchRef:\r\n    name: quickstart\r\nEOF<\/code><\/pre>\n\n\n\n<p> \u53ef\u4ee5\u901a\u8fc7port-forward\u8f6c\u53d1kibana\u7684\u7aef\u53e3\u5230\u5916\u90e8\u8fdb\u884c\u6d4b\u8bd5 <\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>kubectl port-forward service\/quickstart-kb-http 5601\r\n<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">fluentd\u5b89\u88c5<\/h2>\n\n\n\n<p>\u7f16\u5199\u4e00\u4e2afluentd.yaml \uff0c<\/p>\n\n\n\n<p>\u7f16\u5199\u5b8c\u6bd5\u540ekubectl apply -f  fluentd.yaml \u3002\u5185\u5bb9\u5982\u4e0b,\u6ce8\u610f\u66ff\u6362\u5bc6\u7801\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>---\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: fluentd\n  namespace: kube-logging\n  labels:\n    app: fluentd\n---\napiVersion: rbac.authorization.k8s.io\/v1\nkind: ClusterRole\nmetadata:\n  name: fluentd\n  labels:\n    app: fluentd\nrules:\n- apiGroups:\n  - \"\"\n  resources:\n  - pods\n  - namespaces\n  verbs:\n  - get\n  - list\n  - watch\n---\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io\/v1\nmetadata:\n  name: fluentd\nroleRef:\n  kind: ClusterRole\n  name: fluentd\n  apiGroup: rbac.authorization.k8s.io\nsubjects:\n- kind: ServiceAccount\n  name: fluentd\n  namespace: kube-logging\n---\napiVersion: apps\/v1\nkind: DaemonSet\nmetadata:\n  name: fluentd\n  namespace: kube-logging\n  labels:\n    app: fluentd\nspec:\n  selector:\n    matchLabels:\n      app: fluentd\n  template:\n    metadata:\n      labels:\n        app: fluentd\n    spec:\n      serviceAccount: fluentd\n      serviceAccountName: fluentd\n      tolerations:\n      - key: node-role.kubernetes.io\/master\n        effect: NoSchedule\n      containers:\n      - name: fluentd\n        image: fluent\/fluentd-kubernetes-daemonset:v1.4.2-debian-elasticsearch-1.1\n        env:\n          - name:  FLUENT_ELASTICSEARCH_HOST\n            value: \"quickstart-es-http.default.svc.cluster.local\"\n          - name:  FLUENT_ELASTICSEARCH_PORT\n            value: \"9200\"\n          - name: FLUENT_ELASTICSEARCH_SCHEME\n            value: \"http\"\n          - name: FLUENT_ELASTICSEARCH_USER\n            value: \"elastic\"\n          - name: FLUENT_ELASTICSEARCH_PASSWORD\n            value: \"\u6211\u662f\u5bc6\u7801\uff01\u6ce8\u610f\u66ff\u6362\"\n          - name: FLUENT_ELASTICSEARCH_SSL_VERSION\n            value: \"TLSv1_2\"\r\n          - name: FLUENTD_SYSTEMD_CONF\r\n            value: disable\r\n          - name: FLUENT_UID\r\n            value: \"0\"\r\n        resources:\r\n          limits:\r\n            memory: 512Mi\r\n          requests:\r\n            cpu: 100m\r\n            memory: 200Mi\r\n        volumeMounts:\r\n        - name: varlog\r\n          mountPath: \/var\/log\r\n        - name: varlibdockercontainers\r\n          mountPath: \/var\/lib\/docker\/containers\r\n          readOnly: true\r\n      terminationGracePeriodSeconds: 30\r\n      volumes:\r\n      - name: varlog\r\n        hostPath:\r\n          path: \/var\/log\r\n      - name: varlibdockercontainers\r\n        hostPath:\r\n          path: \/var\/lib\/docker\/containers<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">\u90e8\u7f72\u4e00\u4e2a\u6d4b\u8bd5\u7a0b\u5e8f\uff08\u7528\u5b8c\u4e4b\u540e\u53ef\u4ee5\u5220\u9664\uff09<\/h2>\n\n\n\n<pre class=\"wp-block-code\"><code>kubectl -n logging apply -f - &lt;&lt;\"EOF\"\r\napiVersion: apps\/v1\r\nkind: Deployment\r\nmetadata:\r\n name: log-generator\r\nspec:\r\n selector:\r\n   matchLabels:\r\n     app.kubernetes.io\/name: log-generator\r\n replicas: 1\r\n template:\r\n   metadata:\r\n     labels:\r\n       app.kubernetes.io\/name: log-generator\r\n   spec:\r\n     containers:\r\n     - name: nginx\r\n       image: banzaicloud\/log-generator:0.3.2\r\nEOF<\/code><\/pre>\n\n\n\n<h2 class=\"wp-block-heading\">kibana\u91cc\u6dfb\u52a0index\u548c\u67e5\u770b<\/h2>\n\n\n\n<p>\u76f4\u63a5\u770b\u56fe\u8bf4\u8bdd<\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"504\" src=\"https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-1-1024x504.png\" alt=\"\" class=\"wp-image-1036\" srcset=\"https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-1-1024x504.png 1024w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-1-300x148.png 300w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-1-768x378.png 768w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-1-1536x755.png 1536w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-1.png 1903w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"504\" src=\"https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-2-1024x504.png\" alt=\"\" class=\"wp-image-1037\" srcset=\"https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-2-1024x504.png 1024w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-2-300x148.png 300w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-2-768x378.png 768w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-2-1536x755.png 1536w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-2.png 1903w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"504\" src=\"https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-3-1024x504.png\" alt=\"\" class=\"wp-image-1038\" srcset=\"https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-3-1024x504.png 1024w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-3-300x148.png 300w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-3-768x378.png 768w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-3-1536x755.png 1536w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-3.png 1903w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"504\" src=\"https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-4-1024x504.png\" alt=\"\" class=\"wp-image-1039\" srcset=\"https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-4-1024x504.png 1024w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-4-300x148.png 300w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-4-768x378.png 768w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-4-1536x755.png 1536w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-4.png 1903w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<figure class=\"wp-block-image size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"504\" src=\"https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-5-1024x504.png\" alt=\"\" class=\"wp-image-1040\" srcset=\"https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-5-1024x504.png 1024w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-5-300x148.png 300w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-5-768x378.png 768w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-5-1536x755.png 1536w, https:\/\/www.yinyubo.com\/wp-content\/uploads\/2021\/11\/Image-5.png 1903w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<h2 class=\"wp-block-heading\">\u53c2\u8003\u6587\u7ae0<\/h2>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\"><p><\/p><cite>https:\/\/www.digitalocean.com\/community\/tutorials\/how-to-set-up-an-elasticsearch-fluentd-and-kibana-efk-logging-stack-on-kubernetes<br>https:\/\/docs.fluentd.org\/output\/elasticsearch<br>https:\/\/github.com\/fluent\/fluentd-kubernetes-daemonset<br>https:\/\/medium.com\/kubernetes-tutorials\/cluster-level-logging-in-kubernetes-with-fluentd-e59aa2b6093a<\/cite><\/blockquote>\n","protected":false},"excerpt":{"rendered":"<p>\u80cc\u666f \u6700\u8fd1\u60f3\u5b66\u4e60\u4e00\u4e0belasticsearch\u548cfluentd\u7684\u914d\u5408\u4f7f\u7528\uff0c fluentd\u6bd4logstash\u8282 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[],"class_list":["post-1034","post","type-post","status-publish","format-standard","hentry","category-uncategorized"],"_links":{"self":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts\/1034","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1034"}],"version-history":[{"count":3,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts\/1034\/revisions"}],"predecessor-version":[{"id":1079,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=\/wp\/v2\/posts\/1034\/revisions\/1079"}],"wp:attachment":[{"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1034"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1034"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.yinyubo.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1034"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}