测试开发笔记

使用loki+promtail+grafana架构分析nginx日志

Home / 使用loki+promtail+grafana架构分析nginx日志

前端技术

使用loki+promtail+grafana架构分析nginx日志

By 李镇伟 2022年3月14日

0.前提

1.已经阅读过这篇文章https://www.yinyubo.com/2022/03/14/nginx%E5%8A%A8%E6%80%81%E5%A2%9E%E5%8A%A0%E6%A8%A1%E5%9D%97ngx_http_geoip2_module

并且在nginx里已经安装好了geoip2

2.电脑上安装好了docker和docker-compose

1.调整nginx的访问日志格式

编辑/etc/nginx/nginx.conf，内容参考如下

...
load_module modules/ngx_http_geoip2_module.so;
...
http {
    include       /etc/nginx/mime.types;
    geoip2 /home/lzw/GeoLite2-Country_20220222/GeoLite2-Country.mmdb {
        auto_reload 5m;
        $geoip2_metadata_country_build metadata build_epoch;
        $geoip2_data_country_code default=CN source=$remote_addr country iso_code;
        $geoip2_data_country_name country names en;
    }
    geoip2 /home/lzw/GeoLite2-City_20220222/GeoLite2-City.mmdb {
        $geoip2_data_city_name default=Nanjing city names en;
 
    }
    vhost_traffic_status_zone;
    vhost_traffic_status_filter_by_set_key $geoip2_data_country_code country::*;
    log_format json_analytics escape=json '{'
                            '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution
                            '"connection": "$connection", ' # connection serial number
                            '"connection_requests": "$connection_requests", ' # number of requests made in connection
                    '"pid": "$pid", ' # process pid
                    '"request_id": "$request_id", ' # the unique request id
                    '"request_length": "$request_length", ' # request length (including headers and body)
                    '"remote_addr": "$remote_addr", ' # client IP
                    '"remote_user": "$remote_user", ' # client HTTP username
                    '"remote_port": "$remote_port", ' # client port
                    '"time_local": "$time_local", '
                    '"time_iso8601": "$time_iso8601", ' # local time in the ISO 8601 standard format
                    '"request": "$request", ' # full path no arguments if the request
                    '"request_uri": "$request_uri", ' # full path and arguments if the request
                    '"args": "$args", ' # args
                    '"status": "$status", ' # response status code
                    '"body_bytes_sent": "$body_bytes_sent", ' # the number of body bytes exclude headers sent to a client
                    '"bytes_sent": "$bytes_sent", ' # the number of bytes sent to a client
                    '"http_referer": "$http_referer", ' # HTTP referer
                    '"http_user_agent": "$http_user_agent", ' # user agent
                    '"http_x_forwarded_for": "$http_x_forwarded_for", ' # http_x_forwarded_for
                    '"http_host": "$http_host", ' # the request Host: header
                    '"server_name": "$server_name", ' # the name of the vhost serving the request
                    '"request_time": "$request_time", ' # request processing time in seconds with msec resolution
                    '"upstream": "$upstream_addr", ' # upstream backend server for proxied requests
                    '"upstream_connect_time": "$upstream_connect_time", ' # upstream handshake time incl. TLS
                    '"upstream_header_time": "$upstream_header_time", ' # time spent receiving upstream headers
                    '"upstream_response_time": "$upstream_response_time", ' # time spend receiving upstream body
                    '"upstream_response_length": "$upstream_response_length", ' # upstream response length
                    '"upstream_cache_status": "$upstream_cache_status", ' # cache HIT/MISS where applicable
                    '"ssl_protocol": "$ssl_protocol", ' # TLS protocol
                    '"ssl_cipher": "$ssl_cipher", ' # TLS cipher
                    '"scheme": "$scheme", ' # http or https
                    '"request_method": "$request_method", ' # request method
                    '"server_protocol": "$server_protocol", ' # request protocol, like HTTP/1.1 or HTTP/2.0
                    '"pipe": "$pipe", ' # "p" if request was pipelined, "." otherwise
                    '"gzip_ratio": "$gzip_ratio", '
                    '"http_cf_ray": "$http_cf_ray",'
                    '"geoip_country_code": "$geoip2_data_country_code"'
                    '}';
    access_log   /var/log/nginx/json_access.log json_analytics;

2.安装loki+promtail+grafana

1.编写docker-compose文件，内容如下

version: "3"
 
networks:
  loki:
 
services:
  loki:
    image: grafana/loki:2.4.1
    ports:
      - "3100:3100"
    volumes:
      - /home/lzw/loki/loki-conf:/etc/loki
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      - loki
 
  promtail:
    image: grafana/promtail:2.4.1
    volumes:
      - /home/lzw/loki/promtail-conf:/etc/promtail
      - /var/log/nginx:/var/log/nginx
    command: -config.file=/etc/promtail/config.yml
    networks:
      - loki
 
  grafana:
    image: grafana/grafana:latest
    volumes:
      - /home/lzw/loki/grafana:/var/lib/grafana
    ports:
      - "3000:3000"
    networks:
      - loki

2,编写loki-conf/local-config.yaml 配置文件

auth_enabled: false
 
server:
  http_listen_port: 3100
 
common:
  path_prefix: /loki
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: inmemory
 
schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h
 
ruler:
  alertmanager_url: http://localhost:9093

3.编写promtail-conf/config.yml文件

server:
  http_listen_port: 9080
  grpc_listen_port: 0
 
positions:
  filename: /tmp/positions.yaml
 
clients:
  - url: http://loki:3100/loki/api/v1/push
 
scrape_configs:
- job_name: nginx
  static_configs:
  - targets:
      - localhost
    labels:
      job: nginx
      agent: promtail
      __path__: /var/log/nginx/json_access.log

4.准备grafana挂载目录

docker run -d -p 3002:3000 --name=grafana2 grafana/grafana:latest
docker cp grafana2:/var/lib/grafana /home/lzw/loki/.
docker rm -f grafana2
sudo chown -R 472 /home/lzw/loki/grafana

5.运行docker-compose

docker-compose -f docker-compose.yaml up -d

运行完成后，3000端口可以访问grafana、3100端口访问loki。nginx的日志文件通过volume的方式挂载进promtail

3.在grafana里配置报表

1.配置数据源http://loki:3100

2.导入官网模板https://grafana.com/grafana/dashboards/12559

3.导入后的效果应该和下图类似

苏ICP备18047533号-2