Merge pull request '#1001565 针对nginx日志采集' (#1) from pengln/filebeat:master into master

Reviewed-on: https://app.amonstack.com:10301/gitea/ansible-collection/filebeat/pulls/1

playbooks/install_filebeat.yml

@@ -1,53 +1,8 @@
 - name: "Install Filebeat Instance <{{ instance_name }}>"
   hosts: filebeats
 
-  tasks:
-    - name: Install Filebeat
-      ansible.builtin.unarchive:
-        src: "{{ filebeat_download_url }}"
-        dest: "/data/opt/filebeat/{{ instance_name }}"
-        remote_src: yes
-        extra_opts:
-          - --strip-components=1
-
-    - name: Install Supervisor
-      ansible.builtin.pip:
-        name: supervisor
-        executable: "{{ pip_bin_path }}"
-
-    - name: Render Filebeat Configure File
-      ansible.builtin.template:
-        src: filebeat.yml
-        dest: "/data/opt/filebeat/{{ instance_name }}/filebeat.yml"
-        owner: root
-        group: root
-        mode: '0640'
-
-    - name: Render Supervisor Configure File
-      ansible.builtin.template:
-        src: supervisor.yml
-        dest: "/data/opt/filebeat/filebeat_{{ instance_name }}/supervisor.yml"
-        owner: root
-        group: root
-        mode: '0750'
-      vars:
-        work_path: "/data/opt/filebeat/filebeat_{{ instance_name }}"
-
-    - name: Render Supervisor SystemV Script
-      ansible.builtin.template:
-        src: supervisor.sh
-        dest: "/etc/init.d/filebeat_{{ instance_name }}"
-        owner: root
-        group: root
-        mode: '0750'
-      vars:
-        config_file_path: "/data/opt/filebeat/filebeat_{{ instance_name }}/supervisor.yml"
-        work_path: "/data/opt/filebeat/filebeat_{{ instance_name }}"
-
-    - name: Start Filebeat Service
-      ansible.builtin.service:
-        name: "filebeat_{{ instance_name }}"
-        state: restarted
-        enabled: true
+  roles:
+    - filebeat
+    - supervisor
 
 

playbooks/roles/filebeat/.travis.yml

@@ -0,0 +1,29 @@
+---
+language: python
+python: "2.7"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/

playbooks/roles/filebeat/README.md

@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

playbooks/roles/filebeat/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+# defaults file for filebeat

playbooks/roles/filebeat/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for filebeat

playbooks/roles/filebeat/meta/main.yml

@@ -0,0 +1,52 @@
+galaxy_info:
+  author: your name
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.1
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

playbooks/roles/filebeat/tasks/main.yml

@@ -0,0 +1,38 @@
+---
+- name: Make Work directory
+  ansible.builtin.file:
+    path: "{{ work_path }}"
+    owner: filebeat
+    group: filebeat 
+    state: directory
+    mode: '0755'
+
+- name: Create Filebeat Group
+  ansible.builtin.group:
+    name: filebeat
+    state: present
+    gid: 1802
+
+- name: Create Filebeat User
+  ansible.builtin.user:
+    name: filebeat
+    group: filebeat
+    shell: /sbin/nologin
+    state: present
+    uid: 1802
+
+- name: Install Filebeat
+  ansible.builtin.unarchive:
+    src: "{{ filebeat_download_url }}"
+    dest: "{{ work_path }}"
+    remote_src: yes
+    extra_opts:
+      - --strip-components=1
+      
+- name: Render Filebeat Configure File
+  ansible.builtin.template:
+    src: filebeat-gateway-nginx-template.yml
+    dest: "{{ work_path }}/filebeat.yml"
+    owner: root
+    group: root
+    mode: '0644'

playbooks/roles/filebeat/tests/inventory

@@ -0,0 +1,2 @@
+localhost
+

playbooks/roles/filebeat/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - filebeat

playbooks/roles/filebeat/vars/main.yml

@@ -0,0 +1,5 @@
+---
+# vars file for filebeat
+filebeat_download_url: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.10.1-linux-x86_64.tar.gz"
+work_path: "/data/opt/filebeat/{{ instance_name }}"
+logs: ["api", "cl*", "doc", "download", "gw*", "cp", "pay", "static"]

playbooks/roles/supervisor/.travis.yml

@@ -0,0 +1,29 @@
+---
+language: python
+python: "2.7"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/

playbooks/roles/supervisor/README.md

@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

playbooks/roles/supervisor/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+# defaults file for supervisor

playbooks/roles/supervisor/handlers/main.yml

@@ -0,0 +1,2 @@
+---
+# handlers file for supervisor

playbooks/roles/supervisor/meta/main.yml

@@ -0,0 +1,52 @@
+galaxy_info:
+  author: your name
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.1
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

playbooks/roles/supervisor/tasks/main.yml

@@ -0,0 +1,39 @@
+---
+# tasks file for supervisor
+- name: Install Supervisor
+  ansible.builtin.pip:
+    name: supervisor
+    executable: "{{ pip_bin_path }}"
+
+- name: Render Supervisor Configure File
+  ansible.builtin.template:
+    src: supervisor.yml
+    dest: "{{ work_path }}/supervisor.yml"
+    owner: root
+    group: root
+    mode: '0750'
+
+- name: Render Supervisor SystemV Script
+  ansible.builtin.template:
+    src: supervisor.sh
+    dest: "{{ work_path }}/supervisor_initd.sh"
+    owner: root
+    group: root
+    mode: '0750'
+  vars:
+    config_file_path: "{{ work_path }}/supervisor.yml"
+
+- name: Symlink SystemV Script
+  ansible.builtin.file:
+    src: "{{ work_path }}/supervisor_initd.sh"
+    dest: "/etc/init.d/filebeat_{{ instance_name }}"
+    owner: "root"
+    state: link
+
+- name: Start Filebeat Service
+  ansible.builtin.service:
+    name: "filebeat_{{ instance_name }}"
+    state: restarted
+    enabled: true
+    sleep: 30
+    use: sysvinit

playbooks/roles/supervisor/templates/supervisor.sh

@@ -1,11 +1,13 @@
 #!/bin/bash
+# chkconfig:    345 83 04
+
 set -o nounset
 
 . /etc/rc.d/init.d/functions
 
 RETVAL=0
-PIDFILE="{{ work_path }}/supervisord.pid"
-LOCKFILE="{{ work_path }}/supervisord.lock"
+PIDFILE="/tmp/supervisord_{{instance_name}}.pid"
+LOCKFILE="/tmp/supervisord.lock"
 OPTIONS="-c {{ config_file_path }}"
 WAIT_FOR_SUBPROCESSES="yes"
 
@@ -13,7 +15,7 @@ start() {
     echo "Starting supervisord: "
 	if [ -e $PIDFILE ]; then 
 		echo "ALREADY STARTED"
-		return 1
+		return 0
 	fi
 
     {{ python_bin_dir }}/supervisord $OPTIONS
@@ -24,6 +26,7 @@ start() {
 }
 
 stop() {
+    total_sleep=0
     echo -n "Stopping supervisord: "
     {{ python_bin_dir }}/supervisorctl $OPTIONS shutdown
 	if [ -n "$WAIT_FOR_SUBPROCESSES" ]; then 
@@ -33,7 +36,7 @@ stop() {
                 echo "Supervisord exited as expected in under $total_sleep seconds"
                 break
             else
-                if [[ $sleep -eq "last" ]] ; then
+                if [[ $sleep == "last" ]] ; then
                     echo "Supervisord still working on shutting down. We've waited roughly 60 seconds, we'll let it do its thing from here"
                     return 1
                 else
@@ -63,16 +66,12 @@ case "$1" in
         stop
         RETVAL=$?
         ;;
-    restart|force-reload)
-        restart
-        RETVAL=$?
-        ;;
     reload)
         /usr/bin/supervisorctl $OPTIONS reload
         RETVAL=$?
         ;;
-    condrestart)
-        [ -f $LOCKFILE ] && restart
+    restart)
+        [ -f $LOCKFILE ] && restart || start
         RETVAL=$?
         ;;
     status)

playbooks/roles/supervisor/templates/supervisor.yml

@@ -0,0 +1,35 @@
+[supervisord]
+logfile = /tmp/supervisord_{{instance_name}}.log
+logfile_maxbytes = 50MB
+logfile_backups=5
+loglevel = debug
+pidfile = /tmp/supervisord_{{instance_name}}.pid
+nodaemon = false
+minfds = 1024
+minprocs = 200
+umask = 022
+user = filebeat
+identifier = supervisor_{{instance_name}}
+directory = {{ work_path }}
+nocleanup = true
+strip_ansi = false
+
+[unix_http_server]
+file = /tmp/supervisord_{{instance_name}}.sock
+chmod = 0777
+chown = filebeat:filebeat
+username = filebeat
+password = eNlB.UlOrJAnA
+
+[program:filebeat_{{instance_name}}]
+command={{ work_path }}/filebeat -c {{ work_path }}/filebeat.yml
+autorestart=true
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisord_{{instance_name}}.sock
+username = filebeat
+password = eNlB.UlOrJAnA
+prompt = filebeat_{{instance_name}}
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

playbooks/roles/supervisor/tests/inventory

@@ -0,0 +1,2 @@
+localhost
+

playbooks/roles/supervisor/tests/test.yml

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - supervisor

playbooks/roles/supervisor/vars/main.yml

@@ -0,0 +1,6 @@
+---
+# vars file for supervisor
+pip_bin_path: "/data/opt/python3/bin/pip"
+python_bin_dir: "/data/opt/python3/bin"
+work_path: "/data/opt/filebeat/{{ instance_name }}"
+

playbooks/templates/supervisor.yml

@@ -1,30 +0,0 @@
-[supervisord]
-logfile = /var/log/supervisord_{{instance_name}}.log
-logfile_maxbytes = 50MB
-logfile_backups=5
-loglevel = info
-pidfile = {{ work_path }}/supervisord.pid
-nodaemon = false
-minfds = 1024
-minprocs = 200
-umask = 022
-user = filebeat
-identifier = supervisor_{{instance_name}}
-directory = {{ work_path }}
-nocleanup = true
-strip_ansi = false
-
-[unix_http_server]
-file = /tmp/supervisor_{{instance_name}}.sock
-chmod = 0777
-chown= nobody:nogroup
-username = filebeat
-password = eNlB.UlOrJAnA
-
-[program:example]
-
-[supervisorctl]
-serverurl = unix:///tmp/supervisor_{{instance_name}}.sock
-username = filebeat
-password = eNlB.UlOrJAnA
-prompt = supervisor_{{instance_name}}

playbooks/vars/main.yml

@@ -1,4 +0,0 @@
-filebeat_download_url: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.10.1-linux-x86_64.tar.gz"
-pip_bin_path: "/data/opt/python3/bin/pip"
-python_bin_dir: "/data/opt/python3/bin"
-