commit 0f9fdc3320586c80fe437da8c839b19df341582a Author: pengln Date: Wed Nov 3 20:05:07 2021 +0800 first commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..9a7f31d --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Ansible Collection - youyan.filebeat + +Documentation for the collection. diff --git a/galaxy.yml b/galaxy.yml new file mode 100644 index 0000000..cd630dc --- /dev/null +++ b/galaxy.yml @@ -0,0 +1,62 @@ +### REQUIRED +# The namespace of the collection. This can be a company/brand/organization or product namespace under which all +# content lives. May only contain alphanumeric lowercase characters and underscores. Namespaces cannot start with +# underscores or numbers and cannot contain consecutive underscores +namespace: youyan + +# The name of the collection. Has the same character restrictions as 'namespace' +name: filebeat + +# The version of the collection. Must be compatible with semantic versioning +version: 1.7.10 + +# The path to the Markdown (.md) readme file. This path is relative to the root of the collection +readme: README.md + +# A list of the collection's content authors. Can be just the name or in the format 'Full Name (url) +# @nicks:irc/im.site#channel' +authors: +- pengsixiang + + +### OPTIONAL but strongly recommended +# A short summary description of the collection +description: filebeat component install script + +# Either a single license or a list of licenses for content inside of a collection. Ansible Galaxy currently only +# accepts L(SPDX,https://spdx.org/licenses/) licenses. This key is mutually exclusive with 'license_file' +license: +- GPL-2.0-or-later + +# The path to the license file for the collection. This path is relative to the root of the collection. This key is +# mutually exclusive with 'license' +license_file: '' + +# A list of tags you want to associate with the collection for indexing/searching. A tag name has the same character +# requirements as 'namespace' and 'name' +tags: [filebeat, subsystem, youyan] + +# Collections that this collection requires to be installed for it to be usable. The key of the dict is the +# collection label 'namespace.name'. The value is a version range +# L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version +# range specifiers can be set and are separated by ',' +dependencies: {} + +# The URL of the originating SCM repository +repository: https://app.amonstack.com:10301/gitea/pengln/filebeat.git + +# The URL to any online docs +documentation: https://wiki.amonstack.com:10301/zh/ansible-collection/filebeat + +# The URL to the homepage of the collection/project +homepage: https://wiki.amonstack.com:10301/zh/ansible-collection/filebeat + +# The URL to the collection issue tracker +issues: http://youyan.com + +# A list of file glob-like patterns used to filter any files or directories that should not be included in the build +# artifact. A pattern is matched from the relative path of the file or directory of the collection directory. This +# uses 'fnmatch' to match the files or directories. Some directories and files like 'galaxy.yml', '*.pyc', '*.retry', +# and '.git' are always filtered +build_ignore: [] + diff --git a/playbooks/install_filebeat.yml b/playbooks/install_filebeat.yml new file mode 100644 index 0000000..98dddc2 --- /dev/null +++ b/playbooks/install_filebeat.yml @@ -0,0 +1,53 @@ +- name: "Install Filebeat Instance <{{ instance_name }}>" + hosts: filebeats + + tasks: + - name: Install Filebeat + ansible.builtin.unarchive: + src: "{{ filebeat_download_url }}" + dest: "/data/opt/filebeat/{{ instance_name }}" + remote_src: yes + extra_opts: + - --strip-components=1 + + - name: Install Supervisor + ansible.builtin.pip: + name: supervisor + executable: "{{ pip_bin_path }}" + + - name: Render Filebeat Configure File + ansible.builtin.template: + src: filebeat.yml + dest: "/data/opt/filebeat/{{ instance_name }}/filebeat.yml" + owner: root + group: root + mode: '0640' + + - name: Render Supervisor Configure File + ansible.builtin.template: + src: supervisor.yml + dest: "/data/opt/filebeat/filebeat_{{ instance_name }}/supervisor.yml" + owner: root + group: root + mode: '0750' + vars: + work_path: "/data/opt/filebeat/filebeat_{{ instance_name }}" + + - name: Render Supervisor SystemV Script + ansible.builtin.template: + src: supervisor.sh + dest: "/etc/init.d/filebeat_{{ instance_name }}" + owner: root + group: root + mode: '0750' + vars: + config_file_path: "/data/opt/filebeat/filebeat_{{ instance_name }}/supervisor.yml" + work_path: "/data/opt/filebeat/filebeat_{{ instance_name }}" + + - name: Start Filebeat Service + ansible.builtin.service: + name: "filebeat_{{ instance_name }}" + state: restarted + enabled: true + + diff --git a/playbooks/templates/filebeat-gateway-nginx-template.yml b/playbooks/templates/filebeat-gateway-nginx-template.yml new file mode 100644 index 0000000..bdee5c9 --- /dev/null +++ b/playbooks/templates/filebeat-gateway-nginx-template.yml @@ -0,0 +1,51 @@ +filebeat.registry.path: {{ work_path }}/data/registry/gw-nginx-{{ instance_name }} + +filebeat.inputs: +{% for log in logs %} +- type: log + paths: + - "/data/log/nginx/access_{{ log }}.log" + - "/data/log/nginx/error_{{ log }}.log" + fields: + source: {{ log }} + +{% endfor %} + +filebeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s + +# ======================= Elasticsearch template setting ======================= +setup.ilm.enabled: false +setup.template.name: "gw-nginx-ug10cn" +setup.template.pattern: "gw-nginx-ug10cn-*" + +setup.template.settings: + index.number_of_shards: 3 + +setup.kibana: + host: "{{ kibana_url }}" + space.id: "{{ kibana_space }}" + +# ---------------------------- Elasticsearch Output ---------------------------- +output.elasticsearch: + hosts: {{ es_hosts }} + index: "gw-nginx-{{ instance_name }}-%{[fields.source]}-%{+yyyy.MM.dd}" + pipeline: "gwlogs" + username: {{ es_user }} + password: {{ es_passwd }} + + +# ================================= Processors ================================= +processors: + - add_host_metadata: + when.not.contains.tags: forwarded + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ diff --git a/playbooks/templates/supervisor.sh b/playbooks/templates/supervisor.sh new file mode 100644 index 0000000..830b914 --- /dev/null +++ b/playbooks/templates/supervisor.sh @@ -0,0 +1,92 @@ +#!/bin/bash +set -o nounset + +. /etc/rc.d/init.d/functions + +RETVAL=0 +PIDFILE="{{ work_path }}/supervisord.pid" +LOCKFILE="{{ work_path }}/supervisord.lock" +OPTIONS="-c {{ config_file_path }}" +WAIT_FOR_SUBPROCESSES="yes" + +start() { + echo "Starting supervisord: " + if [ -e $PIDFILE ]; then + echo "ALREADY STARTED" + return 1 + fi + + {{ python_bin_dir }}/supervisord $OPTIONS + + {{ python_bin_dir }}/supervisorctl $OPTIONS status + + [ -e $PIDFILE ] && touch $LOCKFILE +} + +stop() { + echo -n "Stopping supervisord: " + {{ python_bin_dir }}/supervisorctl $OPTIONS shutdown + if [ -n "$WAIT_FOR_SUBPROCESSES" ]; then + echo "Waiting roughly 60 seconds for $PIDFILE to be removed after child processes exit" + for sleep in 2 2 2 2 4 4 4 4 8 8 8 8 last; do + if [ ! -e $PIDFILE ] ; then + echo "Supervisord exited as expected in under $total_sleep seconds" + break + else + if [[ $sleep -eq "last" ]] ; then + echo "Supervisord still working on shutting down. We've waited roughly 60 seconds, we'll let it do its thing from here" + return 1 + else + sleep $sleep + total_sleep=$(( $total_sleep + $sleep )) + fi + + fi + done + fi + + # always remove the subsys. we might have waited a while, but just remove it at this point. + rm -f $LOCKFILE +} + +restart() { + stop + start +} + +case "$1" in + start) + start + RETVAL=$? + ;; + stop) + stop + RETVAL=$? + ;; + restart|force-reload) + restart + RETVAL=$? + ;; + reload) + /usr/bin/supervisorctl $OPTIONS reload + RETVAL=$? + ;; + condrestart) + [ -f $LOCKFILE ] && restart + RETVAL=$? + ;; + status) + {{ python_bin_dir }}/supervisorctl $OPTIONS status + status -p $PIDFILE supervisord + # The 'status' option should return one of the LSB-defined return-codes, + # in particular, return-code 3 should mean that the service is not + # currently running. This is particularly important for Ansible's 'service' + # module, as without this behaviour it won't know if a service is up or down. + RETVAL=$? + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|reload|force-reload|condrestart}" + exit 1 +esac + +exit $RETVAL diff --git a/playbooks/templates/supervisor.yml b/playbooks/templates/supervisor.yml new file mode 100644 index 0000000..20892c7 --- /dev/null +++ b/playbooks/templates/supervisor.yml @@ -0,0 +1,30 @@ +[supervisord] +logfile = /var/log/supervisord_{{instance_name}}.log +logfile_maxbytes = 50MB +logfile_backups=5 +loglevel = info +pidfile = {{ work_path }}/supervisord.pid +nodaemon = false +minfds = 1024 +minprocs = 200 +umask = 022 +user = filebeat +identifier = supervisor_{{instance_name}} +directory = {{ work_path }} +nocleanup = true +strip_ansi = false + +[unix_http_server] +file = /tmp/supervisor_{{instance_name}}.sock +chmod = 0777 +chown= nobody:nogroup +username = filebeat +password = eNlB.UlOrJAnA + +[program:example] + +[supervisorctl] +serverurl = unix:///tmp/supervisor_{{instance_name}}.sock +username = filebeat +password = eNlB.UlOrJAnA +prompt = supervisor_{{instance_name}} \ No newline at end of file diff --git a/playbooks/vars/main.yml b/playbooks/vars/main.yml new file mode 100644 index 0000000..bb8adb1 --- /dev/null +++ b/playbooks/vars/main.yml @@ -0,0 +1,4 @@ +filebeat_download_url: "https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.10.1-linux-x86_64.tar.gz" +pip_bin_path: "/data/opt/python3/bin/pip" +python_bin_dir: "/data/opt/python3/bin" + diff --git a/plugins/README.md b/plugins/README.md new file mode 100644 index 0000000..ec2e5ec --- /dev/null +++ b/plugins/README.md @@ -0,0 +1,31 @@ +# Collections Plugins Directory + +This directory can be used to ship various plugins inside an Ansible collection. Each plugin is placed in a folder that +is named after the type of plugin it is in. It can also include the `module_utils` and `modules` directory that +would contain module utils and modules respectively. + +Here is an example directory of the majority of plugins currently supported by Ansible: + +``` +└── plugins + ├── action + ├── become + ├── cache + ├── callback + ├── cliconf + ├── connection + ├── filter + ├── httpapi + ├── inventory + ├── lookup + ├── module_utils + ├── modules + ├── netconf + ├── shell + ├── strategy + ├── terminal + ├── test + └── vars +``` + +A full list of plugin types can be found at [Working With Plugins](https://docs.ansible.com/ansible-core/2.11/plugins/plugins.html).