Loki is a horizontally-scalable, highly-available, multi-tenant log aggregation system inspired by Prometheus. It is designed to be very cost effective and easy to operate. It does not index the contents of the logs, but rather a set of labels for each log stream.

组件介绍

  • Loki:相当于EFK中的ElasticSearch,用于存储日志和处理查询;
  • Promtail:相当于EFK中的 Filebeat/Fluentd,用于采集日志并将其发送给loki ;
  • Grafana:相当于EFK中的Kibana,用于UI展示。

Loki架构

  1. promtail收集并将日志发送给loki的 Distributor 组件
  2. Distributor会对接收到的日志流进行正确性校验,并将验证后的日志分批并行发送到Ingester
  3. Ingester 接受日志流并构建数据块,压缩后存放到所连接的存储后端
  4. Querier 收到HTTP查询请求,并将请求发送至Ingester 用以获取内存数据 ,Ingester 收到请求后返回符合条件的数据;如果 Ingester 没有返回数据,Querier 会从后端存储加载数据并遍历去重执行查询 ,通过HTTP返回查询结果。

轻量日志系统Loki

部署

环境

  • 操作系统:CentOS 8.0.1905
  • loki版本:2.0.0
  • promtail版本:2.0.0
  • grafana版本:7.2.0-1

Docker

$ docker pull "grafana/loki:2.0.0"
$ docker pull "grafana/promtail:2.0.0"

Loki部署

[root@LWS-LADC02-NODE glp]# https://github.com/grafana/loki/releases/download/v2.0.0/loki-linux-amd64.zip
[root@LWS-LADC02-NODE glp]# unzip loki-linux-amd64.zip
[root@LWS-LADC02-NODE glp]# vim loki-local-config.yml
auth_enabled: false
 
server:
  http_listen_port: 3100 # 监听端口
 
ingester:
  lifecycler:
    address: 0.0.0.0 # 监听地址
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 5m
  chunk_retain_period: 30s
  max_transfer_retries: 0
 
schema_config:
  configs:
    - from: 2018-04-15
      store: boltdb
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 144h  #  每张表的时间范围 6天
      chunks:
        period: 144h
 
storage_config:
#  流文件存储地址
  boltdb:
    directory: /tmp/loki/index
#  索引存储地址
  filesystem:
    directory: /tmp/loki/chunks
 
limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 144h
 
chunk_store_config:
  max_look_back_period: 2160h  # 最大可查询历史日期 90天
 

table_manager:   # 表的保留期90天
  retention_deletes_enabled: true
  retention_period: 2160h
[root@LWS-LADC02-NODE glp]# ./loki-linux-amd64 -config.file=./loki-local-config.yml  # 启动loki
[root@LWS-LADC02-NODE glp]# ss -anptu | grep 3100 # 查看端口监听
tcp  LISTEN 0      128                          *:3100                        *:*                                                                                
users:(("loki-linux-amd6",pid=12037,fd=9))   

promtail部署

[root@LWS-LADC02-NODE glp]#  wget https://github.com/grafana/loki/releases/download/v2.0.0/promtail-linux-amd64.zip
[root@LWS-LADC02-NODE glp]#  unzip promtail-linux-amd64.zip
[root@LWS-LADC02-NODE glp]#  vim promtail-local-config.yaml
server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://localhost:3100/loki/api/v1/push

scrape_configs:
- job_name: system
  static_configs:
  - targets:
      - localhost
    labels:
      job: nginx-logs # labels名称
      __path__: /data/wwwlogs/access.log # 采集日志的路径

[root@LWS-LADC02-NODE glp]#   ./promtail-linux-amd64 --config.file=./promtail-local-config.yaml
[root@LWS-LADC02-NODE glp]# ps aux | grep promtail
root     11993  0.2  3.1 1380536 59352 pts/9   Sl+  02:07   0:06 ./promtail-linux-amd64 -config.file=./promtail-local-config.yaml
root     13105  0.0  0.0  12112  1056 pts/10   S+   02:53   0:00 grep --color=auto promtail
[root@LWS-LADC02-NODE glp]# ss -anptu | grep 9080
tcp  LISTEN   0      128                          *:9080                      *:*                                                                                users:(("promtail-linux-",pid=11993,fd=8)) 

grafana部署

[root@LWS-LADC02-NODE glp]# wget https://dl.grafana.com/oss/release/grafana-7.2.0-1.x86_64.rpm
[root@LWS-LADC02-NODE glp]# yum -y install java-1.8.0-openjdk.x86_64
[root@LWS-LADC02-NODE glp]# yum localinstall grafana-7.2.0-1.x86_64.rpm
[root@LWS-LADC02-NODE glp]# systemctl enable grafana-server
[root@LWS-LADC02-NODE glp]# systemctl start grafana-server

测试结果

轻量日志系统Loki
轻量日志系统Loki
轻量日志系统Loki
轻量日志系统Loki
轻量日志系统Loki
轻量日志系统Loki
轻量日志系统Loki
轻量日志系统Loki

参考资料

文章目录