docker-swarm-review

docker-swarm-review/log/README

@@ -0,0 +1,15 @@
+
+# es index生命周期参考 https://developer.aliyun.com/article/793119，为索引配置索引模板即可，索引模板中添加ilm
+# {
+#   "index": {
+#     "lifecycle": {
+#       "name": "90-days-default"
+#     }
+#   }
+# }
+
+
+env $(cat ./env_crm1 | xargs) envsubst < ./docker-compose.yml | docker stack deploy --compose-file - crm1_log   --with-registry-auth
+
+
+env $(cat ./env_review | xargs) envsubst < ./docker-compose.yml | docker stack deploy --compose-file - review_log   --with-registry-auth

docker-swarm-review/log/docker-compose.yml

@@ -0,0 +1,52 @@
+version: '3.8'
+networks:
+  default:
+    name: ${NAMESPACE}
+    external: true
+services:
+  logstash:
+    image: docker.elastic.co/logstash/logstash:8.13.4
+    hostname: ${NAMESPACE}-log-logstash
+    ports:
+      - '${NODE_PORT}:5044'
+    environment:
+      - TZ=Asia/Shanghai
+    configs:
+      - source: logstash_conf
+        target: /usr/share/logstash/pipeline/my.conf
+    logging:
+      driver: json-file
+      options:
+        max-size: "500m"
+        max-file: "3"
+    deploy:
+      placement:
+        constraints:
+          - node.labels.${NAMESPACE}_es==1
+  filebeat:
+    image: docker.elastic.co/beats/filebeat:8.13.4
+    volumes:
+      - "kibana_data:/bitnami/kibana"
+    environment:
+      - TZ=Asia/Shanghai
+      - LOGSTASH_URL=${NAMESPACE}-log-logstash:5044
+      - KIBANA_HOSTS=${NAMESPACE}-es-kibana
+    configs:
+      - source: filebeat_conf
+        target: /usr/share/filebeat/filebeat.yml
+    volumes:
+      - ${NAMESPACE}_logs:/logs
+    deploy:
+      update_config:
+        order: start-first
+      mode: global
+      placement:
+        constraints: [node.platform.os == linux]
+configs:
+  logstash_conf:
+    file: ./logstash.conf
+  filebeat_conf:
+    file: ./filebeat.yml
+volumes:
+  ${NAMESPACE}_logs:
+    external: true

docker-swarm-review/log/env_crm1

@@ -0,0 +1,2 @@
+NAMESPACE=crm1
+NODE_PORT=5045

docker-swarm-review/log/env_review

@@ -0,0 +1,2 @@
+NAMESPACE=review
+NODE_PORT=5044

docker-swarm-review/log/filebeat.yml

@@ -0,0 +1,16 @@
+filebeat.inputs: 
+- type: filestream 
+  id: new-sino-log
+  paths:
+    - "/logs/*/*.log"
+  parsers:
+  - multiline:
+      type: pattern
+      pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+      negate: true
+      match: after
+ 
+setup.kibana.host: "${KIBANA_HOSTS:kibana:5601}"
+ 
+output.logstash:
+  hosts: '${LOGSTASH_URL:logstash:5044}'

docker-swarm-review/log/jvm.options

@@ -0,0 +1,90 @@
+## JVM configuration
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms1g
+-Xmx1g
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+11-13:-XX:+UseConcMarkSweepGC
+11-13:-XX:CMSInitiatingOccupancyFraction=75
+11-13:-XX:+UseCMSInitiatingOccupancyOnly
+
+## Locale
+# Set the locale language
+#-Duser.language=en
+
+# Set the locale country
+#-Duser.country=US
+
+# Set the locale variant, if any
+#-Duser.variant=
+
+## basic
+
+# set the I/O temp directory
+#-Djava.io.tmpdir=$HOME
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+#-Djna.nosys=true
+
+# Turn on JRuby invokedynamic
+-Djruby.compile.invokedynamic=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof
+
+## GC logging
+#-Xlog:gc*,gc+age=trace,safepoint:file=@loggc@:utctime,pid,tags:filecount=32,filesize=64m
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${LS_GC_LOG_FILE}
+
+# Entropy source for randomness
+-Djava.security.egd=file:/dev/urandom
+
+# Copy the logging context from parent threads to children
+-Dlog4j2.isThreadContextMapInheritable=true
+
+# FasterXML/jackson defaults
+#
+# Sets the maximum string length (in chars or bytes, depending on input context).
+# This limit is not exact and an exception will happen at sizes greater than this limit.
+# Some text values that are a little bigger than the limit may be treated as valid but no
+# text values with sizes less than or equal to this limit will be treated as invalid.
+# This value should be higher than `logstash.jackson.stream-read-constraints.max-number-length`.
+# The jackson library defaults to 20000000 or 20MB, whereas Logstash defaults to 200MB or 200000000 characters.
+-Dlogstash.jackson.stream-read-constraints.max-string-length=200000000
+#
+# Sets the maximum number length (in chars or bytes, depending on input context).
+# The jackson library defaults to 1000, whereas Logstash defaults to 10000.
+-Dlogstash.jackson.stream-read-constraints.max-number-length=10000
+#
+# Sets the maximum nesting depth. The depth is a count of objects and arrays that have not
+# been closed, `{` and `[` respectively.
+#-Dlogstash.jackson.stream-read-constraints.max-nesting-depth=1000

docker-swarm-review/log/logstash.conf

@@ -0,0 +1,36 @@
+filter {
+    grok {
+        match => { "message" => "%{TIMESTAMP_ISO8601:oldtimestamp}\s+\[%{DATA:service}\]\s+\[TID:%{NOTSPACE:tid}\]\s+\[%{DATA:thread}\]\s+%{LOGLEVEL:loglevel}\s+%{NOTSPACE:class}\s+-%{GREEDYDATA:oldmessage}"}
+    }
+
+    
+    date {
+      match => ["oldtimestamp", "ISO8601"] 
+      target => "@timestamp" 
+    }
+    mutate {
+      replace => { "message" => "%{oldmessage}" }
+      remove_field => [ "oldmessage","oldtimestamp","agent","host","input","log.flags","log.flags.keyword","tags" ]
+    }
+}
+
+
+output {
+  if [service] {  
+    elasticsearch {
+        hosts => [ "review-es-elasticsearch:9200" ]
+        index => "sslog-%{[service]}"
+        action => "create"
+        ilm_enabled => false
+    }
+  }else{
+    elasticsearch {
+        hosts => [ "review-es-elasticsearch:9200" ]
+        index => "sslog-default"
+        action => "create"
+        ilm_enabled => false
+    }
+  }
+}
+
+