playbooks/roles/filebeat/templates/filebeat-gateway-nginx-template.yml

@@ -1,18 +1,27 @@
 path.data: {{ work_path }}/data
 filebeat.registry.path: {{ work_path }}/data/registry/gw-nginx-{{ instance_name }}
 
+queue.mem:
+  events: 512
+  flush.min_events: 128
+  flush.timeout: 5s
+
+queue.disk:
+  max_size: 10GB
+
 filebeat.inputs:
 {% for log in gwlogs %}
 - type: log
   paths:
-    - "/data/log/nginx/access_{{ log }}.log"
+    - "{{ log }}"
-    - "/data/log/nginx/error_{{ log }}.log"
   fields:
     source: {{ log | regex_replace('\*') }}
   harvester_buffer_size: 1638400
-
+  tail_files: {{ tail_files | default(true) | string | lower }}
 {% endfor %}
 
+max_procs: {{ max_procs | default(2) }}
+
 filebeat.config.modules:
   # Glob pattern for configuration loading
   path: ${path.config}/modules.d/*.yml
@@ -43,7 +52,7 @@ output.elasticsearch:
   username: {{ es_user }}
   password: {{ es_passwd }}
   bulk_max_size: 1500
-  worker: 3
+  worker: 1
 
 
 # ================================= Processors =================================

playbooks/roles/filebeat/templates/filebeat-manager-template.yml

@@ -1,6 +1,14 @@
 path.data: {{ work_path }}/data2
 filebeat.registry.path: {{ work_path }}/data2/registry/mgr-cp-{{ instance_name }}
 
+queue.mem:
+  events: 512
+  flush.min_events: 128
+  flush.timeout: 5s
+
+queue.disk:
+  max_size: 10GB
+  
 filebeat.inputs:
 {% for log in mgrlogs %}
 - type: log
@@ -14,8 +22,11 @@ filebeat.inputs:
   fields:
     source: cp
   harvester_buffer_size: 1638400
+  tail_files: {{ tail_files | default(true) | string | lower }}
 {% endfor %}
 
+max_procs: {{ max_procs | default(2) }}
+
 filebeat.config.modules:
   # Glob pattern for configuration loading
   path: ${path.config}/modules.d/*.yml

playbooks/roles/filebeat/vars/main.yml

@@ -3,15 +3,15 @@
 filebeat_download_url: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.10.1-linux-x86_64.tar.gz"
 work_path: "/data/opt/filebeat/{{ instance_name }}"
 gwlogs:
-  - "api"
+  - "/data/log/nginx/*_api.log"
-  - "cl*"
+  - "/data/log/nginx/*_cl*.log"
-  - "doc"
+  - "/data/log/nginx/*_doc.log"
-  - "download"
+  - "/data/log/nginx/*_download.log"
-  - "gw*"
+  - "/data/log/nginx/*_gw*.log"
-  - "cp*"
+  - "/data/log/nginx/*_cp*.log"
-  - "pay"
+  - "/data/log/nginx/*_pay.log"
-  - "static"
+  - "/data/log/nginx/*_static.log"
-  - "81"
+  - "/data/log/nginx/*_81.log"
 mgrlogs: 
   - "/data/game/htdocs/center/manager/storage/cp/logs/*.log"
 config_files: