Merge pull request '#1001590 增加针对业务应用日志的采集' (#4) from pengln/filebeat:master into master

Reviewed-on: https://app.amonstack.com:10301/gitea/ansible-collection/filebeat/pulls/4

playbooks/roles/filebeat/tasks/main.yml

@@ -31,8 +31,9 @@
       
 - name: Render Filebeat Configure File
   ansible.builtin.template:
-    src: filebeat-gateway-nginx-template.yml
+    src: "filebeat-{{ items }}-template.yml"
-    dest: "{{ work_path }}/filebeat.yml"
+    dest: "{{ work_path }}/filebeat_{{ items }}.yml"
     owner: root
     group: root
     mode: '0644'
+  with_items: "{{ config_files }}"

playbooks/roles/filebeat/templates/filebeat-gateway-nginx-template.yml

@@ -22,9 +22,9 @@ filebeat.config.modules:
   #reload.period: 10s
 
 # ======================= Elasticsearch template setting =======================
-setup.ilm.enabled: false
+setup.ilm.enabled: true
 setup.template.name: "gw-nginx-ug10cn"
-setup.template.pattern: "gw-nginx-ug10cn-*"
+setup.template.pattern: "gw-nginx-*"
 
 setup.template.settings:
   index.number_of_shards: 3

playbooks/roles/filebeat/templates/filebeat-manager-template.yml

@@ -0,0 +1,54 @@
+filebeat.registry.path: {{ work_path }}/data/registry/mgr-cp-{{ instance_name }}
+
+filebeat.inputs:
+{% for log in logs %}
+- type: log
+  paths:
+    - {{ log }}
+  multiline.type: pattern
+  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  multiline.negate: true
+  multiline.match: after
+  multiline.skip_newline: true
+  fields:
+    source: cp
+{% endfor %}
+
+filebeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  #reload.period: 10s
+
+# ======================= Elasticsearch template setting =======================
+setup.ilm.enabled: true
+setup.template.name: "mgr-cp"
+setup.template.pattern: "mgr-cp-*"
+
+setup.template.settings:
+  index.number_of_shards: 3
+
+setup.kibana:
+  host: "{{ kibana_url }}"
+  space.id: "{{ kibana_space }}"
+
+# ---------------------------- Elasticsearch Output ----------------------------
+output.elasticsearch:
+  hosts: {{ es_hosts }}
+  index: "mgr-{{ instance_name }}-%{[fields.source]}-%{+yyyy.MM.dd}"
+  pipeline: "mgrlogs"
+  username: {{ es_user }}
+  password: {{ es_passwd }}
+
+
+# ================================= Processors =================================
+processors:
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
+  - add_cloud_metadata: ~
+  - add_docker_metadata: ~
+  - add_kubernetes_metadata: ~

playbooks/roles/filebeat/vars/main.yml

@@ -2,4 +2,8 @@
 # vars file for filebeat
 filebeat_download_url: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.10.1-linux-x86_64.tar.gz"
 work_path: "/data/opt/filebeat/{{ instance_name }}"
-logs: ["api", "cl*", "doc", "download", "gw*", "cp", "pay", "static"]
+logs: 
+  - "/data/game/htdocs/center/manager/storage/cp/logs/*.log"
+config_files:
+  - "gateway-nginx"
+  - "manager"

playbooks/roles/supervisor/tasks/main.yml

@@ -12,6 +12,8 @@
     owner: root
     group: root
     mode: '0750'
+  vars:
+    items: "{{ filebeat_configs }}"
 
 - name: Render Supervisor SystemV Script
   ansible.builtin.template:

playbooks/roles/supervisor/templates/supervisor.yml

@@ -21,10 +21,13 @@ chown = filebeat:filebeat
 username = filebeat
 password = eNlB.UlOrJAnA
 
-[program:filebeat_{{instance_name}}]
+{% for item in items %}
-command={{ work_path }}/filebeat -c {{ work_path }}/filebeat.yml
+[program:filebeat_{{instance_name}}_{{ item }}]
+command={{ work_path }}/filebeat -c {{ work_path }}/filebeat_{{ item }}.yml
 autorestart=true
 
+{% endfor %}
+
 [supervisorctl]
 serverurl = unix:///tmp/supervisord_{{instance_name}}.sock
 username = filebeat

playbooks/roles/supervisor/vars/main.yml

@@ -3,4 +3,6 @@
 pip_bin_path: "/data/opt/python3/bin/pip"
 python_bin_dir: "/data/opt/python3/bin"
 work_path: "/data/opt/filebeat/{{ instance_name }}"
-
+filebeat_configs:
+  - "gateway-nginx"
+  - "manager"