Создаем такую ирархию
[root@mmb elk]# cat docker-compose.yml
version: '3.7'
services:
elasticsearch:
image: elasticsearch:7.17.4
container_name: elasticsearch
environment:
- xpack.security.enabled=true
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms512m -Xmx512m
- ELASTIC_PASSWORD=12345678
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536
hard: 65536
cap_add:
- IPC_LOCK
volumes:
- elasticsearch-data:/usr/share/elasticsearch/data
- /etc/localtime:/etc/localtime:ro
ports:
- 9200:9200
- 9300:9300
restart: unless-stopped
kibana:
container_name: kibana
image: kibana:7.17.4
environment:
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
- XPACK_MONITORING_ENABLED=false
- XPACK_MONITORING_COLLECTION_ENABLED=false
- XPACK_SECURITY_ENABLED=true
- ELASTICSEARCH_USERNAME=elastic
- ELASTICSEARCH_PASSWORD="12345678"
volumes:
- kibana-data:/usr/share/kibana
- /etc/localtime:/etc/localtime:ro
ports:
- 5601:5601
depends_on:
- elasticsearch
restart: unless-stopped
logstash:
image: logstash:7.17.4
container_name: logstash
environment:
- LS_JAVA_OPTS= -Xms256m -Xmx256m
- XPACK_MONITORING_ENABLED=false
volumes:
- logstash-data:/usr/share/logstash
- /etc/localtime:/etc/localtime:ro
ports:
- 5044:5044
depends_on:
- elasticsearch
restart: unless-stopped
volumes:
elasticsearch-data:
driver: local
logstash-data:
driver: local
kibana-data:
driver: local
[root@mmb elk]#
[root@mmb elk]# docker-compose up -d
Building with native build. Learn about native build in Compose here: https://docs.docker.com/go/compose-native-build/
Creating network "elk_default" with the default driver
Pulling elasticsearch (elasticsearch:7.17.4)...
7.17.4: Pulling from library/elasticsearch
d5fd17ec1767: Pull complete
3aceae0816c1: Pull complete
6f282e391d7d: Pull complete
e0d1c86ab271: Pull complete
1c2d02571b2b: Pull complete
25fb4b01f643: Pull complete
606786004049: Pull complete
28ec7712324b: Pull complete
7d5976c54116: Pull complete
Digest: sha256:529b3cfec4354beda158c6c7f2f8015cbdc9432a48c1d63e824d6fd728f30db2
Status: Downloaded newer image for elasticsearch:7.17.4
Pulling kibana (kibana:7.17.4)...
7.17.4: Pulling from library/kibana
d5fd17ec1767: Already exists
895745324f08: Pull complete
870e4a4d58fd: Pull complete
994a3c1a490e: Pull complete
51ce515a8091: Pull complete
4f4fb700ef54: Pull complete
77156eca877c: Pull complete
6bf02bccac78: Pull complete
02a2ae4c00a5: Pull complete
1111db6cec97: Pull complete
2bbbc44af66d: Pull complete
dee0ccbb5da1: Pull complete
9ad4da5ac21b: Pull complete
e021f342c39d: Pull complete
Digest: sha256:13572cada04ff3730aa7cb6ebc0e0f28e0ae7b4a3a4304fff5104e011b2cba05
Status: Downloaded newer image for kibana:7.17.4
Pulling logstash (logstash:7.17.4)...
7.17.4: Pulling from library/logstash
d5fd17ec1767: Already exists
9448cb0fbc8d: Pull complete
765a92c6bc8c: Pull complete
5bc3b6c90fdc: Pull complete
51db48b30590: Pull complete
7cd65ee8cca0: Pull complete
9f2013c198b5: Pull complete
45ceb6933282: Pull complete
3c143ee9d0a5: Pull complete
961e2d968e85: Pull complete
724d09cc29eb: Pull complete
Digest: sha256:0f97e1c1cd0d990e1126744418a8ae1a45d73cf0067102985322b21ebfc22ac9
Status: Downloaded newer image for logstash:7.17.4
Creating elasticsearch ... done
Creating logstash ... done
Creating kibana ... done
[root@mmb elk]# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
72b24153b011 kibana:7.17.4 "/bin/tini -- /usr/l…" 4 minutes ago Up 4 minutes 0.0.0.0:5601->5601/tcp, :::5601->5601/tcp kibana
c503289481f2 logstash:7.17.4 "/usr/local/bin/dock…" 4 minutes ago Up 4 minutes 0.0.0.0:5044->5044/tcp, :::5044->5044/tcp, 9600/tcp logstash
b9da4db81ee2 elasticsearch:7.17.4 "/bin/tini -- /usr/l…" 4 minutes ago Up 4 minutes 0.0.0.0:9200->9200/tcp, :::9200->9200/tcp, 0.0.0.0:9300->9300/tcp, :::9300->9300/tcp elasticsearch
[root@mmb elk]# cat /etc/filebeat/filebeat.yml
filebeat.prospectors:
- input_type: log
paths:
- /etc/filebeat/log13.json
document_type: system-log
json.keys_under_root: true
json.add_error_key: true
tags: ['system-log']
close_inactive: 24h
output.logstash:
hosts: ["127.0.0.1:5044"]
logging.level: info
logging.to_files: true
logging.files:
path: /var/log/filebeat
name: filebeat
keepfiles: 7
permissions: 0644
[root@mmb elk]#
[root@mmb elk]# cat /etc/filebeat/log13.json
{"timestamp": "2024-01-08T01:20:46.128481", "server": "http_server", "action": "GET", "src_ip": "192.168.100.7", "src_port": 37990, "dest_ip": "0.0.0.0", "dest_port": 80, "cpu_percent": 0.0, "memory_percent": 84.5, "read_bytes": 3698938368, "write_bytes": 8806913024, "bytes_sent": 208632864, "bytes_recv": 1349296354, "packets_sent": 1070273, "packets_recv": 1720369, "Host": "192.168.100.7", "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", "Accept-Language": "ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3", "Accept-Encoding": "gzip, deflate", "Connection": "keep-alive", "Cookie": "sid=Fe26.2**2c1b45aea2f08de118f8fd46e7e1077b6cb3157ce9e32a4aacceb4d34f4c281d*TGpuWWCySt2zA8O2aALRSQ*pKEW4G1nkNrAjtBkMzygVDyU2URy6m-CuLNuIRN7oTQt4EgfNDtqCCgWKPoY0__W2jc3-0U2hgjuzne1Z7oPEHRxNrfIbFt5itLfRo26Z_pXNth7JZlsFphPmGNM6mz8oj4uSVNCT76YFwJcZDs9t4euxtaAdpRuZdCymMC6omKpBSyM4si7uGAxAIbDqFG1Wj4yKL_ORG7j2xI4x0AK_QRMeFSYn5nXrXmGudg9t7k**5c5a794a5650fca5e8a72604d9260c10cfdc1d6d8442b47c5c67f3a12a0c5b9c*7QGhFXs-BxCA4dlkyBnPHfIGTdtgYXPKN2qgqnlofNY", "Upgrade-Insecure-Requests": "1", "method": "GET", "uri": "/"}
[root@mmb config]# pwd
/var/lib/docker/volumes/elk_logstash-data/_data/config
[root@mmb config]# cat pipelines.yml
# This file is where you define your pipelines. You can define multiple.
# For more information on multiple pipelines, see the documentation:
# https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html
- pipeline.id: pipeline_system_logs
path.config: "/usr/share/logstash/pipeline/pipeline_system_logs.conf"
[root@mmb config]# cd ..
[root@mmb _data]# ls -l
итого 48
drwxrwsr-x. 2 1000 root 4096 янв 9 20:03 bin
drwxrwsr-x. 2 1000 root 114 янв 19 23:21 config
-rw-rw-r--. 1 1000 root 2276 мая 18 2022 CONTRIBUTORS
drwxrwsr-x. 4 1000 root 69 янв 9 20:03 data
-rw-rw-r--. 1 1000 root 4050 мая 18 2022 Gemfile
-rw-rw-r--. 1 1000 root 28942 мая 18 2022 Gemfile.lock
drwxrwsr-x. 9 1000 root 121 янв 9 20:03 jdk
drwxrwsr-x. 6 1000 root 84 янв 9 20:03 lib
drwxrwsr-x. 4 1000 root 90 янв 9 20:03 logstash-core
drwxrwsr-x. 3 1000 root 86 янв 9 20:03 logstash-core-plugin-api
drwxrwsr-x. 4 1000 root 55 янв 9 20:03 modules
drwxr-sr-x. 2 1000 root 39 янв 10 05:42 pipeline
drwxrwsr-x. 3 1000 root 30 янв 9 20:03 tools
drwxrwsr-x. 4 1000 root 33 янв 9 20:03 vendor
drwxrwsr-x. 9 1000 root 4096 янв 9 20:03 x-pack
[root@mmb _data]# cd pipeline/
[root@mmb pipeline]# ls -l
итого 4
-rw-r--r--. 1 root root 784 янв 10 00:53 pipeline_system_logs.conf
[root@mmb pipeline]# cat pipeline_system_logs.conf
input { beats { port => 5044 codec => "json" } pipeline { address => "pipeline_system_logs" } }
filter {
if "system-log" in [tags] {
date {
locale => "en"
timezone => "Europe/Moscow"
match => [ "timestamp", "YYYY-MM-dd'T'HH:mm:ss.SSSSSS" ]
target => "@timestamp"
remove_field => "timestamp"
}
json {
source => "message"
skip_on_invalid_json => true
tag_on_failure => ["json_fail"]
}
}
}
output {
if "system-log" in [tags] {
elasticsearch {
hosts => ["http://elasticsearch:9200"]
index => "system-logs-%{+YYYY.MM.dd}"
ssl => false
user => "elastic"
password => "12345678"
timeout => 60
ecs_compatibility => "disabled"
}
}
}
Добавляем этот шаблон через dev tools kibana(добавляем до старта filebeat иначе придется индекс удалять)
[root@mmb pipeline]# cat template.json
PUT _template/template_systemlogs
{
"index_patterns": [
"system-logs-*"
],
"settings": {
"number_of_shards": 1
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"@version": {
"type": "text"
},
"Accept": {
"type": "text"
},
"Accept-Encoding": {
"type": "text"
},
"Accept-Language": {
"type": "text"
},
"Connection": {
"type": "text"
},
"Cookie": {
"type": "text"
},
"Host": {
"type": "text"
},
"Upgrade-Insecure-Requests": {
"type": "text"
},
"User-Agent": {
"type": "text"
},
"action": {
"type": "text"
},
"beat": {
"properties": {
"hostname": {
"type": "text"
},
"name": {
"type": "text"
},
"version": {
"type": "text"
}
}
},
"bytes_recv": {
"type": "long"
},
"bytes_sent": {
"type": "long"
},
"cpu_percent": {
"type": "float"
},
"dest_ip": {
"type": "keyword"
},
"dest_port": {
"type": "integer"
},
"host": {
"properties": {
"name": {
"type": "text"
}
}
},
"log": {
"properties": {
"file": {
"properties": {
"path": {
"type": "text"
}
}
}
}
},
"memory_percent": {
"type": "float"
},
"method": {
"type": "text"
},
"offset": {
"type": "long"
},
"packets_recv": {
"type": "long"
},
"packets_sent": {
"type": "long"
},
"read_bytes": {
"type": "long"
},
"server": {
"type": "text"
},
"source": {
"type": "text"
},
"src_ip": {
"type": "keyword"
},
"src_port": {
"type": "integer"
},
"tags": {
"type": "text"
},
"uri": {
"type": "text"
},
"write_bytes": {
"type": "long"
}
}
},
"aliases": {
"system-logs-hot": {}
}
}
В кибане добавляем индекс system-logs-* или алиас system-logs-hot