playbooks/roles/filebeat/handlers/main.yml

@@ -24,4 +24,8 @@
   when: current_filebeat_version.stdout != ''
 
 - name: Clean Old Filebeat {{ instance_name }}_{{ log_type }}
-  include_tasks: clean_old_filebeat.yml
+  include_tasks: clean_old_filebeat.yml
+
+- name: Restart Filebeat Monitor Plugin {{ instance_name }}_{{ log_type }}
+  ansible.builtin.shell: "service filebeat_{{ instance_name }}_{{ log_type }} restart_monitor"
+  when: current_filebeat_version.stdout != ''

playbooks/roles/filebeat/tasks/main.yml

@@ -42,6 +42,15 @@
     version: "{{ filebeat_logs[log_type]['filebeat_version'] | default(filebeat_version) }}"   
   when: current_filebeat_version.stdout == ''
 
+- name: Install Filebeat Monitor Plugin {{ instance_name }}_{{ log_type }}
+  ansible.builtin.copy:
+    src: filebeat_monitor
+    dest: "{{ filebeat_work_path }}"
+    owner: root
+    group: root
+    mode: '0755'
+  notify: "Restart Filebeat Monitor Plugin {{ instance_name }}_{{ log_type }}"
+
 - name: Render Filebeat Configure File "{{ instance_name }}_{{ log_type }}"
   ansible.builtin.template:
     src: "filebeat.yml"
@@ -65,13 +74,22 @@
   loop_control:
     loop_var: "script"
 
+- name: Render Filebeat Monitor Plugin Configure File "{{ instance_name }}_{{ log_type }}"
+  ansible.builtin.template:
+    src: "filebeat_monitor.yaml"
+    dest: "{{ filebeat_work_path }}/monitor.yaml"
+    owner: root
+    group: root
+    mode: '0644'
+  notify: "Restart Filebeat Monitor Plugin {{ instance_name }}_{{ log_type }}"
+
 - name: Render Filebeat Supervisor Configure File "{{ instance_name }}_{{ log_type }}"
   ansible.builtin.template:
     src: supervisor.yml
     dest: "{{ filebeat_work_path }}/supervisor.yml"
     owner: root
     group: root
-    mode: '0750'
+    mode: '0640'
   notify: "Restart Filebeat Service {{ instance_name }}_{{ log_type }}"
 
 - name: Render Filebeat SystemV Script "{{ instance_name }}_{{ log_type }}"
@@ -97,11 +115,11 @@
 
 - name: Merge Filebeat Data "{{ instance_name }}_{{ log_type }}"
   ansible.builtin.copy:
-    src: "{{ filebeat_logs[log_type]['filebeat_older_dir'] }}/{{ filebeat_logs[log_type]['filebeat_older_data_dir'] }}"
-    dest: "{{ filebeat_work_path }}/data"
+    src: "{{ filebeat_logs[log_type]['filebeat_older_dir'] }}/{{ filebeat_logs[log_type]['filebeat_older_data_dir'] }}/"
+    dest: "{{ filebeat_work_path }}/data/"
     owner: filebeat
     group: filebeat
-    mode: '0755'
+    mode: '0750'
     remote_src: yes
     directory_mode: yes
   notify: "Clean Old Filebeat {{ instance_name }}_{{ log_type }}"
@@ -121,3 +139,7 @@
     sleep: 30
     use: "{{ 'sysvinit' if ansible_facts['distribution'] == 'CentOS' and ansible_facts['distribution_major_version'] == '6' else 'auto' }}"    
   when: filebeat_status.rc != 0
+
+
+
+

playbooks/roles/filebeat/templates/filebeat.yml

@@ -11,6 +11,8 @@ filebeat.inputs:
   ignore_older: {{ filebeat_older }}
   tail_files: {{ filebeat_tail_files | string | lower }}
 
+http.enabled: true
+http.host: unix://${path.config}/filebeat.sock
 max_procs: {{ filebeat_max_procs }}
 
 filebeat.config.modules:
@@ -19,11 +21,11 @@ filebeat.config.modules:
 
 # ======================= Elasticsearch template setting =======================
 setup.ilm.enabled: false
-setup.template.name: "logs[log_type]['template_name']"
-setup.template.pattern: "logs[log_type]['template_pattern']"
+setup.template.name: "{{ logs[log_type]['template_name'] }}"
+setup.template.pattern: "{{ logs[log_type]['template_pattern'] }}"
 
 setup.template.settings:
-  index.number_of_shards: 3
+  index.number_of_shards: "{{ filebeat_number_of_shards | default(3) }}"
 
 {% if 'kibana' in logs[log_type] -%}
 setup.kibana:
@@ -35,16 +37,18 @@ setup.kibana:
 {% if 'output' not in logs[log_type] or logs[log_type]['output'] == 'elastic' %}
 output.elasticsearch:
   hosts: {{ logs[log_type]['es_hosts'] }}
-  {% if 'sublog' in logs[log_type] -%}
+  {% if 'sublog' in logs[log_type] and logs[log_type]['sublog'] -%}
   index: "{{ logs[log_type]['index_prefix'] }}-{{ instance_name }}-{{ logs[log_type]['sublog'] }}-%{+yyyy.MM.dd}"
   {% else -%}
   index: "{{ logs[log_type]['index_prefix'] }}-{{ instance_name }}-%{[sublog]}-%{+yyyy.MM.dd}"
   {% endif -%}
-  {% if 'pipeline' in logs[log_type] -%}
+  {% if 'pipeline' in logs[log_type] and logs[log_type]['pipeline'] -%}
   pipeline: "logs[log_type]['pipeline']"
   {% endif -%}
-  username: {{ logs[log_type]['es_user'] }}
-  password: {{ logs[log_type]['es_passwd'] }}
+  {% if 'es_user' in logs[log_type] and 'es_passwd' in logs[log_type] -%}
+  username: {{ logs[log_type]['es_user'] | default('') }}
+  password: {{ logs[log_type]['es_passwd'] | default('') }}
+  {% endif -%}
 
   bulk_max_size: 1500
   flush_interval: 5s
@@ -64,7 +68,7 @@ processors:
   - add_cloud_metadata: ~
   - add_docker_metadata: ~
   - add_kubernetes_metadata: ~
-  {% if 'dissect_tokenizers' in logs[log_type] -%}
+  {% if 'dissect_tokenizers' in logs[log_type] and logs[log_type]['dissect_tokenizers'] -%}
   {% for tokenizer in logs[log_type]['dissect_tokenizers'] -%}
   - dissect:
       tokenizer: "{{ tokenizer['tokenizer'] }}"
@@ -72,13 +76,13 @@ processors:
       target_prefix: ""  
   {% endfor -%}
   {% endif -%}
-  {% if 'convert_timestamp' in logs[log_type] -%}
+  {% if 'convert_timestamp' in logs[log_type] and logs[log_type]['convert_timestamp'] -%}
   - timestamp:
       field: {{ logs[log_type]['convert_timestamp']['field_name'] }}
       layouts: {{ logs[log_type]['convert_timestamp']['layouts'] }}
       timezone: "Asia/Shanghai"
   {% endif -%}
-  {% if 'scripts' in logs[log_type] -%}
+  {% if 'scripts' in logs[log_type] and logs[log_type]['scripts'] -%}
   {% for script in logs[log_type]['scripts'] -%}
   - script:
       lang: javascript

playbooks/roles/filebeat/templates/filebeat_monitor.yaml

@@ -0,0 +1,10 @@
+project_id: {{ instance_name }}
+sublog: {{ log_type }}
+elastic_hosts: {{ filebeat_logs[log_type]['es_hosts'] }}
+elastic_indice_name: {{ filebeat_logs[log_type]['index_prefix'] }}-{{ instance_name }}-*
+elastic_search_lifecycle: {{ filebeat_monitor_lifecycle | default(30) }}
+http_unix_sock: {{ filebeat_monitor_sock | default('./filebeat_monitor.sock') }}
+prometheus_push_gateway_url: {{ filebeat_monitor_push_gateway | default('') }}
+prometheus_push_gateway_job: {{ filebeat_monitor_push_job | default('filebeat') }}
+prometheus_push_lifecycle: {{ filebeat_monitor_push_lifecycle | default(5) }}
+log_path: {{ filebeat_monitor_log_path | default('logs/filebeat_monitor.log') }}

playbooks/roles/filebeat/templates/supervisor.sh

@@ -1,7 +1,19 @@
 #!/bin/bash
-# chkconfig:    345 83 04
-
-set -o nounset
+# filebeat_{{ instance_name }}_{{ log_type }} Start filebeat
+#
+# chkconfig: 2345 33 81
+# description:  Starts, stops and saves filebeat_{{ instance_name }}_{{ log_type }}
+#
+#
+### BEGIN INIT INFO
+# Provides: youyan
+# Required-Start:
+# Required-Stop:
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: start and stop filebeat_{{ instance_name }}_{{ log_type }}
+# Description: Start, stop filebeat_{{ instance_name }}_{{ log_type }}
+### END INIT INFO
 
 . /etc/rc.d/init.d/functions
 
@@ -70,6 +82,10 @@ case "$1" in
         {{ python_bin_dir }}/supervisorctl $OPTIONS restart filebeat_{{ instance_name }}_{{ log_type }}
         RETVAL=$?
         ;;
+    restart_monitor)
+        {{ python_bin_dir }}/supervisorctl $OPTIONS restart filebeat_monitor_{{ instance_name }}_{{ log_type }}
+        RETVAL=$?
+        ;;
     restart)
         [ -f $LOCKFILE ] && restart || start
         RETVAL=$?

playbooks/roles/filebeat/templates/supervisor.yml

@@ -29,6 +29,14 @@ stderr_logfile=/dev/null
 autostart=true
 autorestart=true
 
+[program:filebeat_monitor_{{ instance_name }}_{{ log_type }}]
+directory={{ filebeat_work_path }}/
+command={{ filebeat_work_path }}/filebeat_monitor
+stdout_logfile=/dev/null
+stderr_logfile=/dev/null
+autostart=true
+autorestart=true
+
 [supervisorctl]
 serverurl = unix:///tmp/supervisord_filebeat_{{ instance_name }}_{{ log_type }}.sock
 username = filebeat_supervisord