From 02b2b99b8ed319e4c02ec347b6356e26058b2d0b Mon Sep 17 00:00:00 2001
From: pengln <pengln@amonstack.com>
Date: Mon, 15 Nov 2021 19:05:45 +0800
Subject: [PATCH 1/2] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86=E9=98=9F?=
 =?UTF-8?q?=E5=88=97=E9=99=90=E5=88=B6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../filebeat-gateway-nginx-template.yml        | 17 +++++++++++++----
 .../templates/filebeat-manager-template.yml    | 11 +++++++++++
 playbooks/roles/filebeat/vars/main.yml         | 18 +++++++++---------
 3 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/playbooks/roles/filebeat/templates/filebeat-gateway-nginx-template.yml b/playbooks/roles/filebeat/templates/filebeat-gateway-nginx-template.yml
index 918a55f..ee190a5 100644
--- a/playbooks/roles/filebeat/templates/filebeat-gateway-nginx-template.yml
+++ b/playbooks/roles/filebeat/templates/filebeat-gateway-nginx-template.yml
@@ -1,18 +1,27 @@
 path.data: {{ work_path }}/data
 filebeat.registry.path: {{ work_path }}/data/registry/gw-nginx-{{ instance_name }}
 
+queue.mem:
+  events: 512
+  flush.min_events: 128
+  flush.timeout: 5s
+
+queue.disk:
+  max_size: 10GB
+
 filebeat.inputs:
 {% for log in gwlogs %}
 - type: log
   paths:
-    - "/data/log/nginx/access_{{ log }}.log"
-    - "/data/log/nginx/error_{{ log }}.log"
+    - "{{ log }}"
   fields:
     source: {{ log | regex_replace('\*') }}
   harvester_buffer_size: 1638400
-
+  tail_files: {{ tail_files | default(true) | string | lower }}
 {% endfor %}
 
+max_procs: {{ max_procs | default(2) }}
+
 filebeat.config.modules:
   # Glob pattern for configuration loading
   path: ${path.config}/modules.d/*.yml
@@ -43,7 +52,7 @@ output.elasticsearch:
   username: {{ es_user }}
   password: {{ es_passwd }}
   bulk_max_size: 1500
-  worker: 3
+  worker: 1
 
 
 # ================================= Processors =================================
diff --git a/playbooks/roles/filebeat/templates/filebeat-manager-template.yml b/playbooks/roles/filebeat/templates/filebeat-manager-template.yml
index 49851df..e63e45d 100644
--- a/playbooks/roles/filebeat/templates/filebeat-manager-template.yml
+++ b/playbooks/roles/filebeat/templates/filebeat-manager-template.yml
@@ -1,6 +1,14 @@
 path.data: {{ work_path }}/data2
 filebeat.registry.path: {{ work_path }}/data2/registry/mgr-cp-{{ instance_name }}
 
+queue.mem:
+  events: 512
+  flush.min_events: 128
+  flush.timeout: 5s
+
+queue.disk:
+  max_size: 10GB
+  
 filebeat.inputs:
 {% for log in mgrlogs %}
 - type: log
@@ -14,8 +22,11 @@ filebeat.inputs:
   fields:
     source: cp
   harvester_buffer_size: 1638400
+  tail_files: {{ tail_files | default(true) | string | lower }}
 {% endfor %}
 
+max_procs: {{ max_procs | default(2) }}
+
 filebeat.config.modules:
   # Glob pattern for configuration loading
   path: ${path.config}/modules.d/*.yml
diff --git a/playbooks/roles/filebeat/vars/main.yml b/playbooks/roles/filebeat/vars/main.yml
index 5959b0c..c2189e9 100644
--- a/playbooks/roles/filebeat/vars/main.yml
+++ b/playbooks/roles/filebeat/vars/main.yml
@@ -3,15 +3,15 @@
 filebeat_download_url: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.10.1-linux-x86_64.tar.gz"
 work_path: "/data/opt/filebeat/{{ instance_name }}"
 gwlogs:
-  - "api"
-  - "cl*"
-  - "doc"
-  - "download"
-  - "gw*"
-  - "cp*"
-  - "pay"
-  - "static"
-  - "81"
+  - "/data/log/nginx/*_api.log"
+  - "/data/log/nginx/*_cl*.log"
+  - "/data/log/nginx/*_doc.log"
+  - "/data/log/nginx/*_download.log"
+  - "/data/log/nginx/*_gw*.log"
+  - "/data/log/nginx/*_cp*.log"
+  - "/data/log/nginx/*_pay.log"
+  - "/data/log/nginx/*_static.log"
+  - "/data/log/nginx/*_81.log"
 mgrlogs: 
   - "/data/game/htdocs/center/manager/storage/cp/logs/*.log"
 config_files:

From 8fb95cbf6acab9d3c344d513fc02eeed7ac8c68e Mon Sep 17 00:00:00 2001
From: pengln <pengln@amonstack.com>
Date: Mon, 15 Nov 2021 19:19:50 +0800
Subject: [PATCH 2/2] =?UTF-8?q?=E5=8F=96=E6=B6=88=E7=A3=81=E7=9B=98?=
 =?UTF-8?q?=E9=98=9F=E5=88=97=E8=A7=A3=E5=86=B3queue=E5=86=B2=E7=AA=81?=
 =?UTF-8?q?=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../filebeat/templates/filebeat-gateway-nginx-template.yml     | 3 ---
 .../roles/filebeat/templates/filebeat-manager-template.yml     | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/playbooks/roles/filebeat/templates/filebeat-gateway-nginx-template.yml b/playbooks/roles/filebeat/templates/filebeat-gateway-nginx-template.yml
index ee190a5..69291c7 100644
--- a/playbooks/roles/filebeat/templates/filebeat-gateway-nginx-template.yml
+++ b/playbooks/roles/filebeat/templates/filebeat-gateway-nginx-template.yml
@@ -6,9 +6,6 @@ queue.mem:
   flush.min_events: 128
   flush.timeout: 5s
 
-queue.disk:
-  max_size: 10GB
-
 filebeat.inputs:
 {% for log in gwlogs %}
 - type: log
diff --git a/playbooks/roles/filebeat/templates/filebeat-manager-template.yml b/playbooks/roles/filebeat/templates/filebeat-manager-template.yml
index e63e45d..d05a8df 100644
--- a/playbooks/roles/filebeat/templates/filebeat-manager-template.yml
+++ b/playbooks/roles/filebeat/templates/filebeat-manager-template.yml
@@ -5,9 +5,6 @@ queue.mem:
   events: 512
   flush.min_events: 128
   flush.timeout: 5s
-
-queue.disk:
-  max_size: 10GB
   
 filebeat.inputs:
 {% for log in mgrlogs %}