first commit

README.md

@@ -0,0 +1,3 @@
+# Ansible Collection - youyan.filebeat
+
+Documentation for the collection.

galaxy.yml

@@ -0,0 +1,62 @@
+### REQUIRED
+# The namespace of the collection. This can be a company/brand/organization or product namespace under which all
+# content lives. May only contain alphanumeric lowercase characters and underscores. Namespaces cannot start with
+# underscores or numbers and cannot contain consecutive underscores
+namespace: youyan
+
+# The name of the collection. Has the same character restrictions as 'namespace'
+name: filebeat
+
+# The version of the collection. Must be compatible with semantic versioning
+version: 1.7.10
+
+# The path to the Markdown (.md) readme file. This path is relative to the root of the collection
+readme: README.md
+
+# A list of the collection's content authors. Can be just the name or in the format 'Full Name <email> (url)
+# @nicks:irc/im.site#channel'
+authors:
+- pengsixiang <pengsixiang82@gmail.com>
+
+
+### OPTIONAL but strongly recommended
+# A short summary description of the collection
+description: filebeat component install script
+
+# Either a single license or a list of licenses for content inside of a collection. Ansible Galaxy currently only
+# accepts L(SPDX,https://spdx.org/licenses/) licenses. This key is mutually exclusive with 'license_file'
+license:
+- GPL-2.0-or-later
+
+# The path to the license file for the collection. This path is relative to the root of the collection. This key is
+# mutually exclusive with 'license'
+license_file: ''
+
+# A list of tags you want to associate with the collection for indexing/searching. A tag name has the same character
+# requirements as 'namespace' and 'name'
+tags: [filebeat, subsystem, youyan]
+
+# Collections that this collection requires to be installed for it to be usable. The key of the dict is the
+# collection label 'namespace.name'. The value is a version range
+# L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version
+# range specifiers can be set and are separated by ','
+dependencies: {}
+
+# The URL of the originating SCM repository
+repository: https://app.amonstack.com:10301/gitea/pengln/filebeat.git
+
+# The URL to any online docs
+documentation: https://wiki.amonstack.com:10301/zh/ansible-collection/filebeat
+
+# The URL to the homepage of the collection/project
+homepage: https://wiki.amonstack.com:10301/zh/ansible-collection/filebeat
+
+# The URL to the collection issue tracker
+issues: http://youyan.com
+
+# A list of file glob-like patterns used to filter any files or directories that should not be included in the build
+# artifact. A pattern is matched from the relative path of the file or directory of the collection directory. This
+# uses 'fnmatch' to match the files or directories. Some directories and files like 'galaxy.yml', '*.pyc', '*.retry',
+# and '.git' are always filtered
+build_ignore: []
+

playbooks/install_filebeat.yml

@@ -0,0 +1,53 @@
+- name: "Install Filebeat Instance <{{ instance_name }}>"
+  hosts: filebeats
+
+  tasks:
+    - name: Install Filebeat
+      ansible.builtin.unarchive:
+        src: "{{ filebeat_download_url }}"
+        dest: "/data/opt/filebeat/{{ instance_name }}"
+        remote_src: yes
+        extra_opts:
+          - --strip-components=1
+
+    - name: Install Supervisor
+      ansible.builtin.pip:
+        name: supervisor
+        executable: "{{ pip_bin_path }}"
+
+    - name: Render Filebeat Configure File
+      ansible.builtin.template:
+        src: filebeat.yml
+        dest: "/data/opt/filebeat/{{ instance_name }}/filebeat.yml"
+        owner: root
+        group: root
+        mode: '0640'
+
+    - name: Render Supervisor Configure File
+      ansible.builtin.template:
+        src: supervisor.yml
+        dest: "/data/opt/filebeat/filebeat_{{ instance_name }}/supervisor.yml"
+        owner: root
+        group: root
+        mode: '0750'
+      vars:
+        work_path: "/data/opt/filebeat/filebeat_{{ instance_name }}"
+
+    - name: Render Supervisor SystemV Script
+      ansible.builtin.template:
+        src: supervisor.sh
+        dest: "/etc/init.d/filebeat_{{ instance_name }}"
+        owner: root
+        group: root
+        mode: '0750'
+      vars:
+        config_file_path: "/data/opt/filebeat/filebeat_{{ instance_name }}/supervisor.yml"
+        work_path: "/data/opt/filebeat/filebeat_{{ instance_name }}"
+
+    - name: Start Filebeat Service
+      ansible.builtin.service:
+        name: "filebeat_{{ instance_name }}"
+        state: restarted
+        enabled: true
+    
+

playbooks/templates/filebeat-gateway-nginx-template.yml

@@ -0,0 +1,51 @@
+filebeat.registry.path: {{ work_path }}/data/registry/gw-nginx-{{ instance_name }}
+
+filebeat.inputs:
+{% for log in logs %}
+- type: log
+  paths:
+    - "/data/log/nginx/access_{{ log }}.log"
+    - "/data/log/nginx/error_{{ log }}.log"
+  fields:
+    source: {{ log }}
+
+{% endfor %}
+
+filebeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  #reload.period: 10s
+
+# ======================= Elasticsearch template setting =======================
+setup.ilm.enabled: false
+setup.template.name: "gw-nginx-ug10cn"
+setup.template.pattern: "gw-nginx-ug10cn-*"
+
+setup.template.settings:
+  index.number_of_shards: 3
+
+setup.kibana:
+  host: "{{ kibana_url }}"
+  space.id: "{{ kibana_space }}"
+
+# ---------------------------- Elasticsearch Output ----------------------------
+output.elasticsearch:
+  hosts: {{ es_hosts }}
+  index: "gw-nginx-{{ instance_name }}-%{[fields.source]}-%{+yyyy.MM.dd}"
+  pipeline: "gwlogs"
+  username: {{ es_user }}
+  password: {{ es_passwd }}
+
+
+# ================================= Processors =================================
+processors:
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
+  - add_cloud_metadata: ~
+  - add_docker_metadata: ~
+  - add_kubernetes_metadata: ~

playbooks/templates/supervisor.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+set -o nounset
+
+. /etc/rc.d/init.d/functions
+
+RETVAL=0
+PIDFILE="{{ work_path }}/supervisord.pid"
+LOCKFILE="{{ work_path }}/supervisord.lock"
+OPTIONS="-c {{ config_file_path }}"
+WAIT_FOR_SUBPROCESSES="yes"
+
+start() {
+    echo "Starting supervisord: "
+	if [ -e $PIDFILE ]; then 
+		echo "ALREADY STARTED"
+		return 1
+	fi
+
+    {{ python_bin_dir }}/supervisord $OPTIONS
+
+	{{ python_bin_dir }}/supervisorctl $OPTIONS status
+	
+    [ -e $PIDFILE ] && touch $LOCKFILE
+}
+
+stop() {
+    echo -n "Stopping supervisord: "
+    {{ python_bin_dir }}/supervisorctl $OPTIONS shutdown
+	if [ -n "$WAIT_FOR_SUBPROCESSES" ]; then 
+        echo "Waiting roughly 60 seconds for $PIDFILE to be removed after child processes exit"
+        for sleep in  2 2 2 2 4 4 4 4 8 8 8 8 last; do
+            if [ ! -e $PIDFILE ] ; then
+                echo "Supervisord exited as expected in under $total_sleep seconds"
+                break
+            else
+                if [[ $sleep -eq "last" ]] ; then
+                    echo "Supervisord still working on shutting down. We've waited roughly 60 seconds, we'll let it do its thing from here"
+                    return 1
+                else
+                    sleep $sleep
+                    total_sleep=$(( $total_sleep + $sleep ))
+                fi
+
+            fi
+        done
+    fi
+
+    # always remove the subsys.  we might have waited a while, but just remove it at this point.
+    rm -f $LOCKFILE
+}
+
+restart() {
+        stop
+        start
+}
+
+case "$1" in
+    start)
+        start
+        RETVAL=$?
+        ;;
+    stop)
+        stop
+        RETVAL=$?
+        ;;
+    restart|force-reload)
+        restart
+        RETVAL=$?
+        ;;
+    reload)
+        /usr/bin/supervisorctl $OPTIONS reload
+        RETVAL=$?
+        ;;
+    condrestart)
+        [ -f $LOCKFILE ] && restart
+        RETVAL=$?
+        ;;
+    status)
+        {{ python_bin_dir }}/supervisorctl $OPTIONS status
+        status -p $PIDFILE supervisord
+        # The 'status' option should return one of the LSB-defined return-codes,
+        # in particular, return-code 3 should mean that the service is not
+        # currently running. This is particularly important for Ansible's 'service'
+        # module, as without this behaviour it won't know if a service is up or down.
+        RETVAL=$?
+        ;;
+    *)
+        echo $"Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}"
+        exit 1
+esac
+
+exit $RETVAL

playbooks/templates/supervisor.yml

@@ -0,0 +1,30 @@
+[supervisord]
+logfile = /var/log/supervisord_{{instance_name}}.log
+logfile_maxbytes = 50MB
+logfile_backups=5
+loglevel = info
+pidfile = {{ work_path }}/supervisord.pid
+nodaemon = false
+minfds = 1024
+minprocs = 200
+umask = 022
+user = filebeat
+identifier = supervisor_{{instance_name}}
+directory = {{ work_path }}
+nocleanup = true
+strip_ansi = false
+
+[unix_http_server]
+file = /tmp/supervisor_{{instance_name}}.sock
+chmod = 0777
+chown= nobody:nogroup
+username = filebeat
+password = eNlB.UlOrJAnA
+
+[program:example]
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisor_{{instance_name}}.sock
+username = filebeat
+password = eNlB.UlOrJAnA
+prompt = supervisor_{{instance_name}}

playbooks/vars/main.yml

@@ -0,0 +1,4 @@
+filebeat_download_url: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.10.1-linux-x86_64.tar.gz"
+pip_bin_path: "/data/opt/python3/bin/pip"
+python_bin_dir: "/data/opt/python3/bin"
+

plugins/README.md

@@ -0,0 +1,31 @@
+# Collections Plugins Directory
+
+This directory can be used to ship various plugins inside an Ansible collection. Each plugin is placed in a folder that
+is named after the type of plugin it is in. It can also include the `module_utils` and `modules` directory that
+would contain module utils and modules respectively.
+
+Here is an example directory of the majority of plugins currently supported by Ansible:
+
+```
+└── plugins
+    ├── action
+    ├── become
+    ├── cache
+    ├── callback
+    ├── cliconf
+    ├── connection
+    ├── filter
+    ├── httpapi
+    ├── inventory
+    ├── lookup
+    ├── module_utils
+    ├── modules
+    ├── netconf
+    ├── shell
+    ├── strategy
+    ├── terminal
+    ├── test
+    └── vars
+```
+
+A full list of plugin types can be found at [Working With Plugins](https://docs.ansible.com/ansible-core/2.11/plugins/plugins.html).