This commit is contained in:
marsal wang
2023-07-26 10:07:34 +08:00
parent f884cb1020
commit 1e5a703cce
5384 changed files with 618283 additions and 4002 deletions

12
docker-swarm/log/README Normal file
View File

@ -0,0 +1,12 @@
# es index生命周期参考 https://developer.aliyun.com/article/793119为索引配置索引模板即可索引模板中添加ilm
# {
# "index": {
# "lifecycle": {
# "name": "90-days-default"
# }
# }
# }
env $(cat ./env_crm1 | xargs) envsubst < ./docker-compose.yml | docker stack deploy --compose-file - crm1_log --with-registry-auth

View File

@ -0,0 +1,46 @@
version: '3.8'
networks:
default:
name: ${NAMESPACE}
external: true
services:
logstash:
image: docker.elastic.co/logstash/logstash:7.17.3
ports:
- '${NODE_PORT}:5044'
environment:
- TZ=Asia/Shanghai
configs:
- source: logstash_conf
target: /usr/share/logstash/pipeline/my.conf
deploy:
placement:
constraints:
- node.labels.${NAMESPACE}_es==1
filebeat:
image: docker.elastic.co/beats/filebeat:7.17.3
volumes:
- "kibana_data:/bitnami/kibana"
environment:
- TZ=Asia/Shanghai
- LOGSTASH_URL=${NAMESPACE}_log_logstash:5044
- KIBANA_HOSTS=${NAMESPACE}_es_kibana
configs:
- source: filebeat_conf
target: /usr/share/filebeat/filebeat.yml
volumes:
- ${NAMESPACE}_logs:/logs
deploy:
update_config:
order: start-first
mode: global
placement:
constraints: [node.platform.os == linux]
configs:
logstash_conf:
file: ./logstash.conf
filebeat_conf:
file: ./filebeat.yml
volumes:
${NAMESPACE}_logs:
external: true

View File

@ -0,0 +1,2 @@
NAMESPACE=crm1
NODE_PORT=5045

View File

@ -0,0 +1,16 @@
filebeat.inputs:
- type: filestream
id: new-sino-log
paths:
- "/logs/*/*.log"
parsers:
- multiline:
type: pattern
pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
negate: true
match: after
setup.kibana.host: "${KIBANA_HOSTS:kibana:5601}"
output.logstash:
hosts: '${LOGSTASH_URL:logstash:5044}'

View File

@ -0,0 +1,36 @@
filter {
grok {
match => { "message" => "%{TIMESTAMP_ISO8601:oldtimestamp}\s+\[%{DATA:service}\]\s+\[TID:%{NOTSPACE:tid}\]\s+\[%{DATA:thread}\]\s+%{LOGLEVEL:loglevel}\s+%{NOTSPACE:class}\s+-%{GREEDYDATA:oldmessage}"}
}
date {
match => ["oldtimestamp", "ISO8601"]
target => "@timestamp"
}
mutate {
replace => { "message" => "%{oldmessage}" }
remove_field => [ "oldmessage","oldtimestamp","agent","host","input","log.flags","log.flags.keyword","tags" ]
}
}
output {
if [servicename] {
elasticsearch {
hosts => [ "crm1-es-elasticsearch:9200" ]
index => "sslog-%{[service]}"
action => "create"
ilm_enabled => false
}
}else{
elasticsearch {
hosts => [ "crm1-es-elasticsearch:9200" ]
index => "sslog-default"
action => "create"
ilm_enabled => false
}
}
}