From bfa03eced0891563ca47dd8955fec4f264a7ffdb Mon Sep 17 00:00:00 2001 From: pengln Date: Wed, 1 Dec 2021 16:12:08 +0800 Subject: [PATCH] =?UTF-8?q?#1001742=20Filebeat=20Ansible=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- galaxy.yml | 2 +- playbooks/deploy.yml | 12 ++ playbooks/install_filebeat.yml | 8 -- playbooks/roles/filebeat/defaults/main.yml | 71 ++++++++--- .../roles/filebeat/files/hb01cn_processor.js | 40 +++++++ .../roles/filebeat/files/hc01cn_processor.js | 42 +++++++ playbooks/roles/filebeat/handlers/main.yml | 25 ++++ playbooks/roles/filebeat/meta/main.yml | 6 +- .../filebeat/tasks/clean_old_filebeat.yml | 17 +++ playbooks/roles/filebeat/tasks/main.yml | 112 +++++++++++++++--- .../filebeat/tasks/stop_old_filebeat.yml | 22 ++++ .../roles/filebeat/templates/filebeat.yml | 88 ++++++++++++++ .../templates/supervisor.sh | 8 +- .../roles/filebeat/templates/supervisor.yml | 39 ++++++ playbooks/roles/supervisor/defaults/main.yml | 5 - playbooks/roles/supervisor/handlers/main.yml | 2 +- playbooks/roles/supervisor/meta/main.yml | 2 + playbooks/roles/supervisor/tasks/main.yml | 32 ----- .../roles/supervisor/templates/supervisor.yml | 38 ------ 19 files changed, 448 insertions(+), 123 deletions(-) create mode 100644 playbooks/deploy.yml delete mode 100644 playbooks/install_filebeat.yml create mode 100644 playbooks/roles/filebeat/files/hb01cn_processor.js create mode 100644 playbooks/roles/filebeat/files/hc01cn_processor.js create mode 100644 playbooks/roles/filebeat/tasks/clean_old_filebeat.yml create mode 100644 playbooks/roles/filebeat/tasks/stop_old_filebeat.yml create mode 100644 playbooks/roles/filebeat/templates/filebeat.yml rename playbooks/roles/{supervisor => filebeat}/templates/supervisor.sh (87%) create mode 100644 playbooks/roles/filebeat/templates/supervisor.yml delete mode 100644 playbooks/roles/supervisor/templates/supervisor.yml diff --git a/galaxy.yml b/galaxy.yml index cd630dc..89a3bda 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -8,7 +8,7 @@ namespace: youyan name: filebeat # The version of the collection. Must be compatible with semantic versioning -version: 1.7.10 +version: 1.8.0 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection readme: README.md diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml new file mode 100644 index 0000000..0cb8842 --- /dev/null +++ b/playbooks/deploy.yml @@ -0,0 +1,12 @@ +- name: "Install Filebeat Instance <{{ instance_name }}>" + hosts: "{{ instance_name }}_filebeats" + + tasks: + - name: Filebeat Deploy + include_role: + name: filebeat + vars: + log_type: "{{ item }}" + with_items: "{{ deploy_logs }}" + + diff --git a/playbooks/install_filebeat.yml b/playbooks/install_filebeat.yml deleted file mode 100644 index 0007fe8..0000000 --- a/playbooks/install_filebeat.yml +++ /dev/null @@ -1,8 +0,0 @@ -- name: "Install Filebeat Instance <{{ instance_name }}>" - hosts: filebeats - - roles: - - filebeat - - supervisor - - diff --git a/playbooks/roles/filebeat/defaults/main.yml b/playbooks/roles/filebeat/defaults/main.yml index 53a7342..9a6cf1e 100644 --- a/playbooks/roles/filebeat/defaults/main.yml +++ b/playbooks/roles/filebeat/defaults/main.yml @@ -1,21 +1,54 @@ --- # vars file for filebeat -filebeat_download_url: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.10.1-linux-x86_64.tar.gz" -work_path: "/data/opt/filebeat/{{ instance_name }}" -gwlogs: - - "api" - - "cl*" - - "doc" - - "download" - - "gw*" - - "cp*" - - "pay" - - "static" - - "81" -mgrlogs: - - "/data/game/htdocs/center/manager/storage/cp/logs/*.log" -quicklog: - - "/data/game/sync/quick/filebeat/*.txt" -config_files: - - "gateway-nginx" - - "manager" \ No newline at end of file +pip_bin_path: "/data/opt/python3/bin/pip" +python_bin_dir: "/data/opt/python3/bin" + +filebeat_version: 7.10.1 +filebeat_oss_version: false +filebeat_download_url: "https://artifacts.elastic.co/downloads/beats/filebeat" +filebeat_work_path: "/data/opt/filebeat/{{ instance_name }}_{{ log_type }}" +filebeat_tail_files: true +filebeat_max_procs: 2 +filebeat_compression_level: 5 +filebeat_output_works: 1 +filebeat_gid: 1801 +filebeat_uid: 1801 +filebeat_older: 24h + +filebeat_logs: + gateway: + paths: + - "/data/log/nginx/*.log" + index_prefix: "gw-nginx" + template_name: "gw-nginx-ug10cn" + template_pattern: "gw-nginx-*" + dissect_tokenizers: + - tokenizer: "/%{}/%{}/%{}/%{}_%{sublog}.log" + field_name: "log.file.path" + filebeat_older_dir: "/data/opt/filebeat/{{ instance_name }}" + filebeat_older_data_dir: data + manager: + paths: + - "/data/game/htdocs/center/manager/storage/cp/logs/*.log" + index_prefix: "mgr-cp" + template_name: "mgr-cp" + template_pattern: "mgr-cp-*" + sublog: cp + filebeat_older_dir: "/data/opt/filebeat/{{ instance_name }}" + filebeat_older_data_dir: data2 + quick: + paths: + - "/data/game/sync/quick/filebeat/*.txt" + index_prefix: "quick" + template_name: "quick" + template_pattern: "quick-*" + sublog: quick + dissect_tokenizers: + - tokenizer: "/%{}/%{}/%{}/%{}/%{}/%{zone_id}_%{server_id}_%{log_file_time}.txt" + field_name: "log.file.path" + convert_timestamp: + field_name: log_datetime + layouts: + - '2006-01-02 15:04:05' + filebeat_older_dir: "/data/opt/filebeat/quick" + filebeat_older_data_dir: data \ No newline at end of file diff --git a/playbooks/roles/filebeat/files/hb01cn_processor.js b/playbooks/roles/filebeat/files/hb01cn_processor.js new file mode 100644 index 0000000..bb38a3a --- /dev/null +++ b/playbooks/roles/filebeat/files/hb01cn_processor.js @@ -0,0 +1,40 @@ +function process(event) { + var msg = event.Get("message"); + var i + + msg = msg.replace("/\r/g", ""); + msg = msg.replace("/\n/g", ""); + msg = msg.replace("/\r/\n/g", ""); + + var fields = msg.split("\t"); + + if (Number(fields[1]) == 64) { + // log_id = 64的日志是子后台同步过来的，需要特殊处理掉 + event.Cancel(); return; + } + + for (i = 0; i < 27; i++) { + if (fields[i] == null) { + fields[i] = ""; + } else if (fields[i].trim() == "-") { + fields[i] = ""; + } else { + fields[i] = fields[i].trim(); + } + } + + event.Put("log_datetime", fields[0]); + event.Put("log_id", fields[1]); + event.Put("account_id", fields[2]); + event.Put("role_id", fields[3]); + event.Put("role_name", fields[4]); + event.Put("spid", fields[5]); + event.Put("channel", fields[6]); + + for (i = 7; i < fields.length; i++) { + if (fields[i] == "-") { fields[i] = "" } + event.Put("col" + (i - 6), fields[i]); + } + + return event; +} \ No newline at end of file diff --git a/playbooks/roles/filebeat/files/hc01cn_processor.js b/playbooks/roles/filebeat/files/hc01cn_processor.js new file mode 100644 index 0000000..7ed1e22 --- /dev/null +++ b/playbooks/roles/filebeat/files/hc01cn_processor.js @@ -0,0 +1,42 @@ +function process(event) { + var msg = event.Get("message"); + var i + + msg = msg.replace("/\r/g", ""); + msg = msg.replace("/\n/g", ""); + msg = msg.replace("/\r/\n/g", ""); + + var fields = msg.split("\t"); + + if (Number(fields[1]) == 64) { + // log_id = 64的日志是子后台同步过来的，需要特殊处理掉 + event.Cancel(); return; + } + + for (i = 0; i < 25; i++) { + if (fields[i] == null) { + fields[i] = ""; + } else if (fields[i].trim() == "-") { + fields[i] = ""; + } else { + fields[i] = fields[i].trim(); + } + } + + event.Put("log_datetime", fields[0]); + event.Put("log_id", fields[1]); + event.Put("role_id", fields[2]); + event.Put("role_name", fields[3]); + event.Put("account_id", fields[4]); + + var ws = fields[4].split("_") // 从account_id分离spid和channel + // account_id中只有第一个部分是描述spid的，故spid和channel使用相同的信息填充 + event.Put("spid", ws[0]); + event.Put("channel", ws[0]); + + for (i = 5; i < fields.length; i++) { + event.Put("col" + (i - 4), fields[i]); + } + + return event; +} \ No newline at end of file diff --git a/playbooks/roles/filebeat/handlers/main.yml b/playbooks/roles/filebeat/handlers/main.yml index 40c1d20..761d994 100644 --- a/playbooks/roles/filebeat/handlers/main.yml +++ b/playbooks/roles/filebeat/handlers/main.yml @@ -1,2 +1,27 @@ --- # handlers file for filebeat +- name: Reload Filebeat Service {{ instance_name }}_{{ log_type }} + ansible.builtin.service: + name: "filebeat_{{ instance_name }}_{{ log_type }}" + state: reloaded + use: "{{ 'sysvinit' if ansible_facts['distribution'] == 'CentOS' and ansible_facts['distribution_major_version'] == '6' else 'auto' }}" + when: current_filebeat_version.stdout != '' + +- name: Start Filebeat Service {{ instance_name }}_{{ log_type }} + ansible.builtin.service: + name: "filebeat_{{ instance_name }}_{{ log_type }}" + state: started + enabled: true + sleep: 5 + use: "{{ 'sysvinit' if ansible_facts['distribution'] == 'CentOS' and ansible_facts['distribution_major_version'] == '6' else 'auto' }}" + +- name: Restart Filebeat Service {{ instance_name }}_{{ log_type }} + ansible.builtin.service: + name: "filebeat_{{ instance_name }}_{{ log_type }}" + state: restarted + sleep: 5 + use: "{{ 'sysvinit' if ansible_facts['distribution'] == 'CentOS' and ansible_facts['distribution_major_version'] == '6' else 'auto' }}" + when: current_filebeat_version.stdout != '' + +- name: Clean Old Filebeat {{ instance_name }}_{{ log_type }} + include_tasks: clean_old_filebeat.yml \ No newline at end of file diff --git a/playbooks/roles/filebeat/meta/main.yml b/playbooks/roles/filebeat/meta/main.yml index c572acc..6c04042 100644 --- a/playbooks/roles/filebeat/meta/main.yml +++ b/playbooks/roles/filebeat/meta/main.yml @@ -47,6 +47,10 @@ galaxy_info: # NOTE: A tag is limited to a single word comprised of alphanumeric characters. # Maximum 20 tags per role. -dependencies: [] +dependencies: + - supervisor + # List your role dependencies here, one per line. Be sure to remove the '[]' above, # if you add dependencies to this list. + +allow_duplicates: true diff --git a/playbooks/roles/filebeat/tasks/clean_old_filebeat.yml b/playbooks/roles/filebeat/tasks/clean_old_filebeat.yml new file mode 100644 index 0000000..ee42a11 --- /dev/null +++ b/playbooks/roles/filebeat/tasks/clean_old_filebeat.yml @@ -0,0 +1,17 @@ +--- +- name: Check Old Filebeat Directory + ansible.builtin.shell: "ls {{ filebeat_logs[log_type]['filebeat_older_dir'] }} || echo ''" + register: check_older_dir + changed_when: check_older_dir.stdout != '' + +- name: Clean Old Filebeat Directory + community.general.archive: + path: "{{ filebeat_logs[log_type]['filebeat_older_dir'] }}" + dest: "/data/old-filebeat-{{ 'quick' if log_type == 'quick' else instance_name }}-backup.tgz" + remove: yes + when: check_older_dir.stdout != '' + +- name: Clean Old Filebeat SysV Script + ansible.builtin.shell: "/bin/rm -vf /etc/init.d/filebeat_{{ instance_name }}" + register: clean_sysv_script + changed_when: clean_sysv_script.stdout != '' diff --git a/playbooks/roles/filebeat/tasks/main.yml b/playbooks/roles/filebeat/tasks/main.yml index 2ebd7eb..110b411 100644 --- a/playbooks/roles/filebeat/tasks/main.yml +++ b/playbooks/roles/filebeat/tasks/main.yml @@ -1,39 +1,123 @@ --- -- name: Create Filebeat Group +- name: Create Filebeat Group "{{ instance_name }}_{{ log_type }}" ansible.builtin.group: name: filebeat state: present - gid: 1802 + gid: "{{ filebeat_gid }}" -- name: Create Filebeat User +- name: Create Filebeat User "{{ instance_name }}_{{ log_type }}" ansible.builtin.user: name: filebeat group: filebeat shell: /sbin/nologin state: present - uid: 1802 + uid: "{{ filebeat_uid }}" -- name: Make Work directory +- name: Make Work directory "{{ instance_name }}_{{ log_type }}" ansible.builtin.file: - path: "{{ work_path }}" + path: "{{ filebeat_work_path }}" owner: filebeat group: filebeat state: directory mode: '0755' -- name: Install Filebeat +- name: Check Filebeat Version "{{ instance_name }}_{{ log_type }}" + ansible.builtin.shell: "({{ filebeat_work_path }}/filebeat version | grep -E 'filebeat version {{ version }}') || echo ''" + register: current_filebeat_version + changed_when: current_filebeat_version.stdout == '' + vars: + version: "{{ filebeat_logs[log_type]['filebeat_version'] | default(filebeat_version) }}" + +- name: Install Filebeat "{{ instance_name }}_{{ log_type }} {{ 'filebeat-oss-' if filebeat_logs[log_type]['filebeat_oss_version'] | default(filebeat_oss_version) else 'filebeat-' }}{{ filebeat_logs[log_type]['filebeat_version'] | default(filebeat_version) }}" ansible.builtin.unarchive: - src: "{{ filebeat_download_url }}" - dest: "{{ work_path }}" + src: "{{ filebeat_logs[log_type]['filebeat_download_url'] | default(filebeat_download_url) }}/{{ package_name }}" + dest: "{{ filebeat_work_path }}" remote_src: yes + validate_certs: false extra_opts: - --strip-components=1 - -- name: Render Filebeat Configure File + vars: + package_name: "{{ 'filebeat-oss' if oss else 'filebeat' }}-{{ version }}-{{ ansible_facts['system'] | lower }}-{{ ansible_facts['userspace_architecture'] }}.tar.gz" + oss: "{{ filebeat_logs[log_type]['filebeat_oss_version'] | default(filebeat_oss_version) }}" + version: "{{ filebeat_logs[log_type]['filebeat_version'] | default(filebeat_version) }}" + when: current_filebeat_version.stdout == '' + +- name: Render Filebeat Configure File "{{ instance_name }}_{{ log_type }}" ansible.builtin.template: - src: "filebeat-{{ item }}-template.yml" - dest: "{{ work_path }}/filebeat_{{ item }}.yml" + src: "filebeat.yml" + dest: "{{ filebeat_work_path }}/filebeat.yml" owner: root group: root mode: '0644' - with_items: "{{ config_files }}" \ No newline at end of file + notify: "Reload Filebeat Service {{ instance_name }}_{{ log_type }}" + vars: + logs: "{{ filebeat_logs }}" + +- name: Render Custom Processor "{{ instance_name }}_{{ log_type }}" + ansible.builtin.copy: + src: "{{ script['file'] }}" + dest: "{{ filebeat_work_path }}" + owner: root + group: root + mode: '0644' + notify: "Reload Filebeat Service {{ instance_name }}_{{ log_type }}" + with_items: "{{ filebeat_logs[log_type]['scripts'] | default([]) }}" + loop_control: + loop_var: "script" + +- name: Render Filebeat Supervisor Configure File "{{ instance_name }}_{{ log_type }}" + ansible.builtin.template: + src: supervisor.yml + dest: "{{ filebeat_work_path }}/supervisor.yml" + owner: root + group: root + mode: '0750' + notify: "Restart Filebeat Service {{ instance_name }}_{{ log_type }}" + +- name: Render Filebeat SystemV Script "{{ instance_name }}_{{ log_type }}" + ansible.builtin.template: + src: supervisor.sh + dest: "/etc/init.d/filebeat_{{ instance_name }}_{{ log_type }}" + owner: root + group: root + mode: '0750' + notify: "Restart Filebeat Service {{ instance_name }}_{{ log_type }}" + +- name: Stop Old Filebeat "{{ instance_name }}_{{ log_type }}" + include_tasks: stop_old_filebeat.yml + vars: + target: "{{ instance_name }}_{{ log_type }}" + when: filebeat_data_migration | default(false) + +- name: Check Old Filebeat Data "{{ instance_name }}_{{ log_type }}" + ansible.builtin.shell: "ls {{ filebeat_logs[log_type]['filebeat_older_dir'] }}/{{ filebeat_logs[log_type]['filebeat_older_data_dir'] }} || echo ''" + register: check_old_filebeat_data + changed_when: check_old_filebeat_data.stdout != '' + when: filebeat_data_migration | default(false) + +- name: Merge Filebeat Data "{{ instance_name }}_{{ log_type }}" + ansible.builtin.copy: + src: "{{ filebeat_logs[log_type]['filebeat_older_dir'] }}/{{ filebeat_logs[log_type]['filebeat_older_data_dir'] }}" + dest: "{{ filebeat_work_path }}/data" + owner: filebeat + group: filebeat + mode: '0755' + remote_src: yes + directory_mode: yes + notify: "Clean Old Filebeat {{ instance_name }}_{{ log_type }}" + when: filebeat_data_migration | default(false) and check_old_filebeat_data.stdout != '' + +- name: Check Filebeat Service Running Status + ansible.builtin.shell: "supervisorctl -c {{ filebeat_work_path }}/supervisor.yml status" + register: filebeat_status + changed_when: filebeat_status.rc != 0 + failed_when: false + +- name: First Start Filebeat Service "{{ instance_name }}_{{ log_type }}" + ansible.builtin.service: + name: "filebeat_{{ instance_name }}_{{ log_type }}" + state: started + enabled: true + sleep: 30 + use: "{{ 'sysvinit' if ansible_facts['distribution'] == 'CentOS' and ansible_facts['distribution_major_version'] == '6' else 'auto' }}" + when: filebeat_status.rc != 0 diff --git a/playbooks/roles/filebeat/tasks/stop_old_filebeat.yml b/playbooks/roles/filebeat/tasks/stop_old_filebeat.yml new file mode 100644 index 0000000..d108bc2 --- /dev/null +++ b/playbooks/roles/filebeat/tasks/stop_old_filebeat.yml @@ -0,0 +1,22 @@ +--- +- name: Get Old Running Supervisor Processes {{ target }} + ansible.builtin.shell: "ps -ef | grep -v grep | grep -w /data/opt/filebeat/{{ instance_name }}/supervisor.yml | awk '{print $2}'" + register: supervisor_processes + changed_when: supervisor_processes.stdout != '' + +- name: Get Old Running Filebeat Processes {{ target }} + ansible.builtin.shell: "ps -ef | grep -v grep | grep -w /data/opt/filebeat/{{ instance_name }}/filebeat | awk '{print $2}'" + register: filebeat_processes + changed_when: filebeat_processes.stdout != '' + +- name: Stop Old Filebeat Processes {{ target }} + ansible.builtin.shell: "kill -9 {{ (supervisor_processes.stdout_lines + filebeat_processes.stdout_lines) | join(' ') }}" + when: supervisor_processes.stdout != '' or filebeat_processes.stdout != '' + +- name: Remove Old Processes Residue + ansible.builtin.shell: "/bin/rm -vf /tmp/supervisord_{{ instance_name }}.* /tmp/filebeat_{{ target }}*.log" + register: residue + changed_when: residue.stdout != '' + + + diff --git a/playbooks/roles/filebeat/templates/filebeat.yml b/playbooks/roles/filebeat/templates/filebeat.yml new file mode 100644 index 0000000..c0372e3 --- /dev/null +++ b/playbooks/roles/filebeat/templates/filebeat.yml @@ -0,0 +1,88 @@ +filebeat.config.inputs: + enabled: true + path: ${path.config}/filebeat_quick.yml + reload.enabled: true + reload.period: 10s + +filebeat.inputs: +- type: log + paths: {{ logs[log_type]['paths'] }} + harvester_buffer_size: 40960 + ignore_older: {{ filebeat_older }} + tail_files: {{ filebeat_tail_files | string | lower }} + +max_procs: {{ filebeat_max_procs }} + +filebeat.config.modules: + path: ${path.config}/modules.d/*.yml + reload.enabled: false + +# ======================= Elasticsearch template setting ======================= +setup.ilm.enabled: false +setup.template.name: "logs[log_type]['template_name']" +setup.template.pattern: "logs[log_type]['template_pattern']" + +setup.template.settings: + index.number_of_shards: 3 + +{% if 'kibana' in logs[log_type] -%} +setup.kibana: + host: "{{ logs[log_type]['kibana']['kibana_url'] }}" + space.id: "{{ logs[log_type]['kibana']['kibana_space'] }}" +{% endif -%} + +# ---------------------------- Elasticsearch Output ---------------------------- +{% if 'output' not in logs[log_type] or logs[log_type]['output'] == 'elastic' %} +output.elasticsearch: + hosts: {{ logs[log_type]['es_hosts'] }} + {% if 'sublog' in logs[log_type] -%} + index: "{{ logs[log_type]['index_prefix'] }}-{{ instance_name }}-{{ logs[log_type]['sublog'] }}-%{+yyyy.MM.dd}" + {% else -%} + index: "{{ logs[log_type]['index_prefix'] }}-{{ instance_name }}-%{[sublog]}-%{+yyyy.MM.dd}" + {% endif -%} + {% if 'pipeline' in logs[log_type] -%} + pipeline: "logs[log_type]['pipeline']" + {% endif -%} + username: {{ logs[log_type]['es_user'] }} + password: {{ logs[log_type]['es_passwd'] }} + + bulk_max_size: 1500 + flush_interval: 5s + compression_level: {{ filebeat_compression_level }} + worker: {{ filebeat_output_works }} +{% endif -%} + +{% if 'output' in logs[log_type] and logs[log_type]['output'] == 'debug' -%} +output.console: + pretty: true +{% endif -%} + +# ================================= Processors ================================= +processors: + - add_host_metadata: + when.not.contains.tags: forwarded + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ + {% if 'dissect_tokenizers' in logs[log_type] -%} + {% for tokenizer in logs[log_type]['dissect_tokenizers'] -%} + - dissect: + tokenizer: "{{ tokenizer['tokenizer'] }}" + field: {{ tokenizer['field_name'] }} + target_prefix: "" + {% endfor -%} + {% endif -%} + {% if 'convert_timestamp' in logs[log_type] -%} + - timestamp: + field: {{ logs[log_type]['convert_timestamp']['field_name'] }} + layouts: {{ logs[log_type]['convert_timestamp']['layouts'] }} + timezone: "Asia/Shanghai" + {% endif -%} + {% if 'scripts' in logs[log_type] -%} + {% for script in logs[log_type]['scripts'] -%} + - script: + lang: javascript + id: {{ script['id'] }} + file: {{ script['file'] }} + {% endfor %} + {%- endif -%} \ No newline at end of file diff --git a/playbooks/roles/supervisor/templates/supervisor.sh b/playbooks/roles/filebeat/templates/supervisor.sh similarity index 87% rename from playbooks/roles/supervisor/templates/supervisor.sh rename to playbooks/roles/filebeat/templates/supervisor.sh index f59089a..6a86d47 100644 --- a/playbooks/roles/supervisor/templates/supervisor.sh +++ b/playbooks/roles/filebeat/templates/supervisor.sh @@ -6,9 +6,9 @@ set -o nounset . /etc/rc.d/init.d/functions RETVAL=0 -PIDFILE="/tmp/supervisord_{{instance_name}}.pid" -LOCKFILE="/tmp/supervisord.lock" -OPTIONS="-c {{ config_file_path }}" +PIDFILE="/tmp/supervisord_filebeat_{{ instance_name }}_{{ log_type }}.pid" +LOCKFILE="/tmp/supervisord_filebeat_{{ instance_name }}_{{ log_type }}.lock" +OPTIONS="-c {{ filebeat_work_path }}/supervisor.yml" WAIT_FOR_SUBPROCESSES="yes" start() { @@ -67,7 +67,7 @@ case "$1" in RETVAL=$? ;; reload) - /usr/bin/supervisorctl $OPTIONS reload + {{ python_bin_dir }}/supervisorctl $OPTIONS restart filebeat_{{ instance_name }}_{{ log_type }} RETVAL=$? ;; restart) diff --git a/playbooks/roles/filebeat/templates/supervisor.yml b/playbooks/roles/filebeat/templates/supervisor.yml new file mode 100644 index 0000000..23415cd --- /dev/null +++ b/playbooks/roles/filebeat/templates/supervisor.yml @@ -0,0 +1,39 @@ +[supervisord] +logfile = /tmp/supervisord_filebeat_{{ instance_name }}_{{ log_type }}.log +logfile_maxbytes = 50MB +logfile_backups=5 +loglevel = debug +pidfile = /tmp/supervisord_filebeat_{{ instance_name }}_{{ log_type }}.pid +nodaemon = false +minfds = 1024 +minprocs = 200 +umask = 022 +user = filebeat +identifier = filebeat_{{ instance_name }}_{{ log_type }} +directory = {{ filebeat_work_path }} +nocleanup = true +strip_ansi = false + +[unix_http_server] +file = /tmp/supervisord_filebeat_{{ instance_name }}_{{ log_type }}.sock +chmod = 0777 +chown = filebeat:filebeat +username = filebeat_supervisord +password = eNlB.UlOrJAnA + +[program:filebeat_{{ instance_name }}_{{ log_type }}] +directory={{ filebeat_work_path }}/ +command={{ filebeat_work_path }}/filebeat -c {{ filebeat_work_path }}/filebeat.yml +stdout_logfile=/dev/null +stderr_logfile=/dev/null +autostart=true +autorestart=true + +[supervisorctl] +serverurl = unix:///tmp/supervisord_filebeat_{{ instance_name }}_{{ log_type }}.sock +username = filebeat_supervisord +password = eNlB.UlOrJAnA +prompt = filebeat_{{ instance_name }}_{{ log_type }} + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface \ No newline at end of file diff --git a/playbooks/roles/supervisor/defaults/main.yml b/playbooks/roles/supervisor/defaults/main.yml index 4326286..ddd367e 100644 --- a/playbooks/roles/supervisor/defaults/main.yml +++ b/playbooks/roles/supervisor/defaults/main.yml @@ -2,8 +2,3 @@ # vars file for supervisor pip_bin_path: "/data/opt/python3/bin/pip" python_bin_dir: "/data/opt/python3/bin" -work_path: "/data/opt/filebeat/{{ instance_name }}" -filebeat_configs: - - "gateway-nginx" - - "manager" - - "quick" diff --git a/playbooks/roles/supervisor/handlers/main.yml b/playbooks/roles/supervisor/handlers/main.yml index c9b6637..e05c8b1 100644 --- a/playbooks/roles/supervisor/handlers/main.yml +++ b/playbooks/roles/supervisor/handlers/main.yml @@ -1,2 +1,2 @@ --- -# handlers file for supervisor +# handlers file for supervisor \ No newline at end of file diff --git a/playbooks/roles/supervisor/meta/main.yml b/playbooks/roles/supervisor/meta/main.yml index c572acc..8cbdabe 100644 --- a/playbooks/roles/supervisor/meta/main.yml +++ b/playbooks/roles/supervisor/meta/main.yml @@ -50,3 +50,5 @@ galaxy_info: dependencies: [] # List your role dependencies here, one per line. Be sure to remove the '[]' above, # if you add dependencies to this list. + +allow_duplicates: true \ No newline at end of file diff --git a/playbooks/roles/supervisor/tasks/main.yml b/playbooks/roles/supervisor/tasks/main.yml index 8b2747c..5e0102e 100644 --- a/playbooks/roles/supervisor/tasks/main.yml +++ b/playbooks/roles/supervisor/tasks/main.yml @@ -5,37 +5,5 @@ name: supervisor executable: "{{ pip_bin_path }}" -- name: Render Supervisor Configure File - ansible.builtin.template: - src: supervisor.yml - dest: "{{ work_path }}/supervisor.yml" - owner: root - group: root - mode: '0750' - vars: - items: "{{ filebeat_configs }}" -- name: Render Supervisor SystemV Script - ansible.builtin.template: - src: supervisor.sh - dest: "{{ work_path }}/supervisor_initd.sh" - owner: root - group: root - mode: '0750' - vars: - config_file_path: "{{ work_path }}/supervisor.yml" -- name: Symlink SystemV Script - ansible.builtin.file: - src: "{{ work_path }}/supervisor_initd.sh" - dest: "/etc/init.d/filebeat_{{ instance_name }}" - owner: "root" - state: link - -- name: Start Filebeat Service - ansible.builtin.service: - name: "filebeat_{{ instance_name }}" - state: restarted - enabled: true - sleep: 30 - use: sysvinit \ No newline at end of file diff --git a/playbooks/roles/supervisor/templates/supervisor.yml b/playbooks/roles/supervisor/templates/supervisor.yml deleted file mode 100644 index 16edf0a..0000000 --- a/playbooks/roles/supervisor/templates/supervisor.yml +++ /dev/null @@ -1,38 +0,0 @@ -[supervisord] -logfile = /tmp/supervisord_{{instance_name}}.log -logfile_maxbytes = 50MB -logfile_backups=5 -loglevel = debug -pidfile = /tmp/supervisord_{{instance_name}}.pid -nodaemon = false -minfds = 1024 -minprocs = 200 -umask = 022 -user = filebeat -identifier = supervisor_{{instance_name}} -directory = {{ work_path }} -nocleanup = true -strip_ansi = false - -[unix_http_server] -file = /tmp/supervisord_{{instance_name}}.sock -chmod = 0777 -chown = filebeat:filebeat -username = filebeat -password = eNlB.UlOrJAnA - -{% for item in items %} -[program:filebeat_{{instance_name}}_{{ item }}] -command={{ work_path }}/filebeat -c {{ work_path }}/filebeat_{{ item }}.yml -autorestart=true - -{% endfor %} - -[supervisorctl] -serverurl = unix:///tmp/supervisord_{{instance_name}}.sock -username = filebeat -password = eNlB.UlOrJAnA -prompt = filebeat_{{instance_name}} - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface \ No newline at end of file